2024-09-20T23:40:21.554 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-20T23:40:21.576 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T23:40:21.666 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7913105' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 workunit: branch: squid sha1: 4e45e6b490e6cbe082cdf155fea780438ec53fc7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 9344 sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 sleep_before_teardown: 0 subset: 56/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4e45e6b490e6cbe082cdf155fea780438ec53fc7 targets: smithi018.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKnLFll97zQCElB4AnLsgAI8aZgLVkeJ3mTrATK6RA+dd+ffqBVugjoje8cXbPkVjNt12lxTfOqLKYLxFmATwDQ= smithi031.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAMW458zG3mfIewSwxZo5Zk5bS5kw4ZmiYVc/gzZWY1eqEooc3M0n+KNpO2Gs2IexjrTxBP0LjN8vEWMz0rR43E= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-20_21:08:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-20T23:40:21.666 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa; will attempt to use it 2024-09-20T23:40:21.667 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4e45e6b490e6cbe082cdf155fea780438ec53fc7/qa/tasks 2024-09-20T23:40:21.667 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-20T23:40:21.669 INFO:teuthology.task.internal:Checking packages... 2024-09-20T23:40:21.689 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '9e2ac70bb7aea67d34f0d50002344abbe7803736' 2024-09-20T23:40:21.689 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-20T23:40:21.690 INFO:teuthology.packaging:ref: None 2024-09-20T23:40:21.690 INFO:teuthology.packaging:tag: None 2024-09-20T23:40:21.690 INFO:teuthology.packaging:branch: squid 2024-09-20T23:40:21.690 INFO:teuthology.packaging:sha1: 9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T23:40:21.690 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-20T23:40:21.932 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-373.g4e45e6b4 2024-09-20T23:40:21.934 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-20T23:40:21.941 INFO:teuthology.task.internal:no buildpackages task found 2024-09-20T23:40:21.941 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-20T23:40:21.951 INFO:teuthology.task.internal:Saving configuration 2024-09-20T23:40:21.968 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-20T23:40:21.974 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-20T23:40:21.997 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi018.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 23:35:42.304219', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKnLFll97zQCElB4AnLsgAI8aZgLVkeJ3mTrATK6RA+dd+ffqBVugjoje8cXbPkVjNt12lxTfOqLKYLxFmATwDQ='} 2024-09-20T23:40:22.021 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi031.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 23:35:42.302261', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAMW458zG3mfIewSwxZo5Zk5bS5kw4ZmiYVc/gzZWY1eqEooc3M0n+KNpO2Gs2IexjrTxBP0LjN8vEWMz0rR43E='} 2024-09-20T23:40:22.021 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-20T23:40:22.029 INFO:teuthology.task.internal:roles: ubuntu@smithi018.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-20T23:40:22.030 INFO:teuthology.task.internal:roles: ubuntu@smithi031.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-20T23:40:22.030 INFO:teuthology.run_tasks:Running task console_log... 2024-09-20T23:40:22.089 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4b9213c160>, signals=[15]) 2024-09-20T23:40:22.089 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-20T23:40:22.100 INFO:teuthology.task.internal:Opening connections... 2024-09-20T23:40:22.100 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi018.front.sepia.ceph.com 2024-09-20T23:40:22.101 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:40:22.178 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi031.front.sepia.ceph.com 2024-09-20T23:40:22.179 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:40:22.255 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-20T23:40:22.262 DEBUG:teuthology.orchestra.run.smithi018:> uname -m 2024-09-20T23:40:22.281 INFO:teuthology.orchestra.run.smithi018.stdout:x86_64 2024-09-20T23:40:22.281 DEBUG:teuthology.orchestra.run.smithi018:> cat /etc/os-release 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:NAME="CentOS Stream" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION="9" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:ID="centos" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:ID_LIKE="rhel fedora" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION_ID="9" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:PLATFORM_ID="platform:el9" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T23:40:22.339 INFO:teuthology.orchestra.run.smithi018.stdout:ANSI_COLOR="0;31" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:LOGO="fedora-logo-icon" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:HOME_URL="https://centos.org/" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T23:40:22.340 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T23:40:22.340 INFO:teuthology.lock.ops:Updating smithi018.front.sepia.ceph.com on lock server 2024-09-20T23:40:22.362 DEBUG:teuthology.orchestra.run.smithi031:> uname -m 2024-09-20T23:40:22.380 INFO:teuthology.orchestra.run.smithi031.stdout:x86_64 2024-09-20T23:40:22.381 DEBUG:teuthology.orchestra.run.smithi031:> cat /etc/os-release 2024-09-20T23:40:22.438 INFO:teuthology.orchestra.run.smithi031.stdout:NAME="CentOS Stream" 2024-09-20T23:40:22.438 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION="9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:ID="centos" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:ID_LIKE="rhel fedora" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION_ID="9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:PLATFORM_ID="platform:el9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:ANSI_COLOR="0;31" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:LOGO="fedora-logo-icon" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:HOME_URL="https://centos.org/" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T23:40:22.439 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T23:40:22.440 INFO:teuthology.lock.ops:Updating smithi031.front.sepia.ceph.com on lock server 2024-09-20T23:40:22.462 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-20T23:40:22.471 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-20T23:40:22.476 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-20T23:40:22.477 DEBUG:teuthology.orchestra.run.smithi018:> test '!' -e /home/ubuntu/cephtest 2024-09-20T23:40:22.479 DEBUG:teuthology.orchestra.run.smithi031:> test '!' -e /home/ubuntu/cephtest 2024-09-20T23:40:22.496 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-20T23:40:22.502 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-20T23:40:22.502 DEBUG:teuthology.orchestra.run.smithi018:> test -z $(ls -A /var/lib/ceph) 2024-09-20T23:40:22.537 DEBUG:teuthology.orchestra.run.smithi031:> test -z $(ls -A /var/lib/ceph) 2024-09-20T23:40:22.564 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-20T23:40:22.618 INFO:teuthology.run_tasks:Running task kernel... 2024-09-20T23:40:22.633 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-20T23:40:22.633 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-20T23:40:22.634 DEBUG:teuthology.orchestra.run.smithi018:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T23:40:22.634 DEBUG:teuthology.orchestra.run.smithi031:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T23:40:22.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T23:40:22.651 DEBUG:teuthology.orchestra.run.smithi018:> uname -r 2024-09-20T23:40:22.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T23:40:22.652 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2024-09-20T23:40:22.709 INFO:teuthology.orchestra.run.smithi018.stdout:5.14.0-503.el9.x86_64 2024-09-20T23:40:22.709 INFO:teuthology.task.kernel:Running kernel on smithi018: 5.14.0-503.el9.x86_64 2024-09-20T23:40:22.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum install -y kernel 2024-09-20T23:40:22.710 INFO:teuthology.orchestra.run.smithi031.stdout:5.14.0-503.el9.x86_64 2024-09-20T23:40:22.710 INFO:teuthology.task.kernel:Running kernel on smithi031: 5.14.0-503.el9.x86_64 2024-09-20T23:40:22.710 DEBUG:teuthology.orchestra.run.smithi031:> sudo yum install -y kernel 2024-09-20T23:40:24.966 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.3 MB 00:00 2024-09-20T23:40:24.980 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-20T23:40:26.984 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - AppStream 23 MB/s | 20 MB 00:00 2024-09-20T23:40:29.514 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 9 - AppStream 6.0 MB/s | 20 MB 00:03 2024-09-20T23:40:31.793 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 6.5 MB 00:00 2024-09-20T23:40:33.703 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 19 kB 00:00 2024-09-20T23:40:34.656 INFO:teuthology.orchestra.run.smithi018.stdout:Extra Packages for Enterprise Linux 28 MB/s | 23 MB 00:00 2024-09-20T23:40:35.539 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 9 - CRB 3.1 MB/s | 6.5 MB 00:02 2024-09-20T23:40:37.265 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 19 kB 00:00 2024-09-20T23:40:38.052 INFO:teuthology.orchestra.run.smithi031.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-20T23:40:40.195 INFO:teuthology.orchestra.run.smithi018.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-20T23:40:42.185 INFO:teuthology.orchestra.run.smithi018.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T23:40:42.185 INFO:teuthology.orchestra.run.smithi018.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T23:40:42.278 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2024-09-20T23:40:42.292 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-20T23:40:42.292 INFO:teuthology.orchestra.run.smithi018.stdout: Package Architecture Version Repository Size 2024-09-20T23:40:42.292 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-20T23:40:42.292 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout:Install 4 Packages 2024-09-20T23:40:42.293 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:40:42.294 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 79 M 2024-09-20T23:40:42.294 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 126 M 2024-09-20T23:40:42.294 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2024-09-20T23:40:42.905 INFO:teuthology.orchestra.run.smithi018.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 114 kB/s | 36 kB 00:00 2024-09-20T23:40:43.674 INFO:teuthology.orchestra.run.smithi031.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-20T23:40:44.230 INFO:teuthology.orchestra.run.smithi018.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 9.5 MB/s | 16 MB 00:01 2024-09-20T23:40:44.980 INFO:teuthology.orchestra.run.smithi018.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 14 MB/s | 29 MB 00:02 2024-09-20T23:40:45.464 INFO:teuthology.orchestra.run.smithi018.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 12 MB/s | 35 MB 00:02 2024-09-20T23:40:45.464 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2024-09-20T23:40:45.464 INFO:teuthology.orchestra.run.smithi018.stdout:Total 25 MB/s | 79 MB 00:03 2024-09-20T23:40:45.606 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T23:40:45.606 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T23:40:45.699 INFO:teuthology.orchestra.run.smithi031.stdout:Dependencies resolved. 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout: Package Architecture Version Repository Size 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout:Installing: 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout:Installing dependencies: 2024-09-20T23:40:45.715 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction Summary 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout:Install 4 Packages 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:40:45.716 INFO:teuthology.orchestra.run.smithi031.stdout:Total download size: 79 M 2024-09-20T23:40:45.717 INFO:teuthology.orchestra.run.smithi031.stdout:Installed size: 126 M 2024-09-20T23:40:45.717 INFO:teuthology.orchestra.run.smithi031.stdout:Downloading Packages: 2024-09-20T23:40:46.059 INFO:teuthology.orchestra.run.smithi031.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 122 kB/s | 36 kB 00:00 2024-09-20T23:40:46.177 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2024-09-20T23:40:46.557 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2024-09-20T23:40:46.557 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2024-09-20T23:40:47.143 INFO:teuthology.orchestra.run.smithi031.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-20T23:40:47.565 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2024-09-20T23:40:47.565 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2024-09-20T23:40:47.951 INFO:teuthology.orchestra.run.smithi031.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 16 MB/s | 35 MB 00:02 2024-09-20T23:40:48.676 INFO:teuthology.orchestra.run.smithi031.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 11 MB/s | 29 MB 00:02 2024-09-20T23:40:48.676 INFO:teuthology.orchestra.run.smithi031.stdout:-------------------------------------------------------------------------------- 2024-09-20T23:40:48.676 INFO:teuthology.orchestra.run.smithi031.stdout:Total 27 MB/s | 79 MB 00:02 2024-09-20T23:40:49.384 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction check 2024-09-20T23:40:49.771 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction check succeeded. 2024-09-20T23:40:49.771 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction test 2024-09-20T23:40:50.136 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2024-09-20T23:40:50.751 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction test succeeded. 2024-09-20T23:40:50.752 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction 2024-09-20T23:40:50.800 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T23:40:50.889 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:40:53.307 INFO:teuthology.orchestra.run.smithi031.stdout: Preparing : 1/1 2024-09-20T23:40:53.644 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:40:53.776 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:40:53.965 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T23:40:54.046 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:40:56.801 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:40:56.932 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:40:58.904 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:40:58.943 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:02.061 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:41:02.108 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:07.248 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:43.017 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:43.058 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:43.360 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:43.360 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T23:41:43.361 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:41:43.361 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:41:44.167 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2024-09-20T23:41:44.523 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T23:41:44.611 INFO:teuthology.orchestra.run.smithi018.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.612 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T23:41:44.612 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T23:41:44.612 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T23:41:44.613 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T23:41:44.613 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-20T23:41:44.613 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi018.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T23:41:44.613 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T23:41:44.672 INFO:teuthology.orchestra.run.smithi018.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:44.672 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T23:41:44.672 DEBUG:teuthology.orchestra.run.smithi018:> sudo rpm -qi grub2-tools 2024-09-20T23:41:44.706 INFO:teuthology.orchestra.run.smithi018.stdout:Name : grub2-tools 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Epoch : 1 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Version : 2.06 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Release : 82.el9 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Architecture: x86_64 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Group : Unspecified 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Size : 8274670 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:License : GPLv3+ 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T23:41:44.707 INFO:teuthology.orchestra.run.smithi018.stdout:Packager : builder@centos.org 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:Vendor : CentOS 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:Summary : Support tools for GRUB. 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:Description : 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:hardware devices. 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:41:44.708 INFO:teuthology.orchestra.run.smithi018.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T23:41:44.709 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T23:41:44.709 INFO:teuthology.task.kernel:Updating grub on smithi018 to boot 5.14.0-508.el9.x86_64 2024-09-20T23:41:44.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T23:41:45.314 INFO:teuthology.orchestra.run.smithi018.stderr:Generating grub configuration file ... 2024-09-20T23:41:46.080 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:46.121 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:46.598 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:46.598 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T23:41:46.599 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T23:41:46.599 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T23:41:46.676 INFO:teuthology.orchestra.run.smithi018.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T23:41:46.704 INFO:teuthology.orchestra.run.smithi018.stderr:done 2024-09-20T23:41:46.716 DEBUG:teuthology.orchestra.run.smithi018:> mktemp 2024-09-20T23:41:46.731 INFO:teuthology.orchestra.run.smithi018.stdout:/tmp/tmp.RzPnWDISt1 2024-09-20T23:41:46.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.RzPnWDISt1 2024-09-20T23:41:46.797 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 0666 /tmp/tmp.RzPnWDISt1 2024-09-20T23:41:46.934 DEBUG:teuthology.orchestra.remote:smithi018:/tmp/tmp.RzPnWDISt1 is 6KB 2024-09-20T23:41:46.982 DEBUG:teuthology.orchestra.run.smithi018:> rm -fr /tmp/tmp.RzPnWDISt1 2024-09-20T23:41:46.997 DEBUG:teuthology.orchestra.run.smithi018:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T23:41:47.063 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T23:41:47.063 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T23:41:47.063 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T23:41:47.063 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T23:41:47.064 DEBUG:teuthology.orchestra.run.smithi018:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.216 DEBUG:teuthology.orchestra.run.smithi018:> sudo shutdown -r now 2024-09-20T23:41:47.443 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout:Installed: 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:41:47.444 INFO:teuthology.orchestra.run.smithi031.stdout:Complete! 2024-09-20T23:41:47.799 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T23:41:47.890 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.891 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T23:41:47.891 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T23:41:47.891 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T23:41:47.891 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T23:41:47.891 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-20T23:41:47.891 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi031.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T23:41:47.891 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T23:41:47.949 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T23:41:47.949 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T23:41:47.950 DEBUG:teuthology.orchestra.run.smithi031:> sudo rpm -qi grub2-tools 2024-09-20T23:41:47.980 INFO:teuthology.orchestra.run.smithi031.stdout:Name : grub2-tools 2024-09-20T23:41:47.980 INFO:teuthology.orchestra.run.smithi031.stdout:Epoch : 1 2024-09-20T23:41:47.980 INFO:teuthology.orchestra.run.smithi031.stdout:Version : 2.06 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Release : 82.el9 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Architecture: x86_64 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Group : Unspecified 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Size : 8274670 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:License : GPLv3+ 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Packager : builder@centos.org 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:Vendor : CentOS 2024-09-20T23:41:47.981 INFO:teuthology.orchestra.run.smithi031.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:Summary : Support tools for GRUB. 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:Description : 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:hardware devices. 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:41:47.982 INFO:teuthology.orchestra.run.smithi031.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T23:41:47.983 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T23:41:47.983 INFO:teuthology.task.kernel:Updating grub on smithi031 to boot 5.14.0-508.el9.x86_64 2024-09-20T23:41:47.983 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T23:41:48.692 INFO:teuthology.orchestra.run.smithi031.stderr:Generating grub configuration file ... 2024-09-20T23:41:49.984 INFO:teuthology.orchestra.run.smithi031.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T23:41:50.006 INFO:teuthology.orchestra.run.smithi031.stderr:done 2024-09-20T23:41:50.007 DEBUG:teuthology.orchestra.run.smithi031:> mktemp 2024-09-20T23:41:50.022 INFO:teuthology.orchestra.run.smithi031.stdout:/tmp/tmp.AoOgMBXYlt 2024-09-20T23:41:50.023 DEBUG:teuthology.orchestra.run.smithi031:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.AoOgMBXYlt 2024-09-20T23:41:50.087 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 0666 /tmp/tmp.AoOgMBXYlt 2024-09-20T23:41:50.348 DEBUG:teuthology.orchestra.remote:smithi031:/tmp/tmp.AoOgMBXYlt is 6KB 2024-09-20T23:41:50.396 DEBUG:teuthology.orchestra.run.smithi031:> rm -fr /tmp/tmp.AoOgMBXYlt 2024-09-20T23:41:50.411 DEBUG:teuthology.orchestra.run.smithi031:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T23:41:50.473 INFO:teuthology.orchestra.run.smithi031.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T23:41:50.474 INFO:teuthology.orchestra.run.smithi031.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T23:41:50.474 INFO:teuthology.orchestra.run.smithi031.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T23:41:50.474 INFO:teuthology.orchestra.run.smithi031.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T23:41:50.475 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T23:41:50.742 DEBUG:teuthology.orchestra.run.smithi031:> sudo shutdown -r now 2024-09-20T23:42:17.245 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-20T23:42:17.246 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:42:17.246 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:42:20.746 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-20T23:42:20.747 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:42:20.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:42:35.759 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.18 2024-09-20T23:42:39.087 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.31 2024-09-20T23:42:44.766 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:42:44.766 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:42:48.090 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:42:48.091 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:43:06.483 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.31 2024-09-20T23:43:18.494 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:43:18.495 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:43:19.728 DEBUG:teuthology.orchestra.run.smithi031:> true 2024-09-20T23:43:20.102 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:43:20.103 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-20T23:43:20.103 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2024-09-20T23:43:20.120 INFO:teuthology.orchestra.run.smithi031.stdout:5.14.0-508.el9.x86_64 2024-09-20T23:43:20.120 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T23:43:20.120 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T23:43:20.120 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T23:43:21.121 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-20T23:43:21.122 DEBUG:teuthology.orchestra.run.smithi031:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T23:43:21.215 INFO:teuthology.orchestra.run.smithi031.stdout:ttyS1 2024-09-20T23:43:21.282 DEBUG:teuthology.parallel:result is None 2024-09-20T23:43:44.769 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T23:43:56.770 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:43:56.771 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:43:57.118 DEBUG:teuthology.orchestra.run.smithi018:> true 2024-09-20T23:43:57.492 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:43:57.493 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-20T23:43:57.493 DEBUG:teuthology.orchestra.run.smithi018:> uname -r 2024-09-20T23:43:57.509 INFO:teuthology.orchestra.run.smithi018.stdout:5.14.0-508.el9.x86_64 2024-09-20T23:43:57.509 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T23:43:57.510 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T23:43:57.510 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T23:43:58.510 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-20T23:43:58.511 DEBUG:teuthology.orchestra.run.smithi018:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T23:43:58.579 INFO:teuthology.orchestra.run.smithi018.stdout:ttyS1 2024-09-20T23:43:58.600 DEBUG:teuthology.parallel:result is None 2024-09-20T23:43:58.600 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-20T23:43:58.607 INFO:teuthology.task.internal:Creating test directory... 2024-09-20T23:43:58.607 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T23:43:58.610 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T23:43:58.632 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-20T23:43:58.639 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-20T23:43:58.648 INFO:teuthology.task.internal:Creating archive directory... 2024-09-20T23:43:58.648 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T23:43:58.668 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T23:43:58.702 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-20T23:43:58.709 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-20T23:43:58.709 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T23:43:58.732 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T23:43:58.767 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T23:43:58.779 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T23:43:58.781 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T23:43:58.793 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T23:43:58.794 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-20T23:43:58.807 INFO:teuthology.task.internal:Configuring sudo... 2024-09-20T23:43:58.808 DEBUG:teuthology.orchestra.run.smithi018:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T23:43:58.823 DEBUG:teuthology.orchestra.run.smithi031:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T23:43:58.863 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-20T23:43:58.871 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-20T23:43:58.872 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T23:43:58.892 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T23:43:58.920 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T23:43:58.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T23:43:59.039 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T23:43:59.122 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T23:43:59.166 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:43:59.166 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T23:43:59.233 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T23:43:59.279 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T23:43:59.331 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T23:43:59.414 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T23:43:59.459 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:43:59.460 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T23:43:59.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo service rsyslog restart 2024-09-20T23:43:59.529 DEBUG:teuthology.orchestra.run.smithi031:> sudo service rsyslog restart 2024-09-20T23:43:59.572 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T23:43:59.615 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T23:44:00.049 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-20T23:44:00.057 INFO:teuthology.task.internal:Starting timer... 2024-09-20T23:44:00.057 INFO:teuthology.run_tasks:Running task pcp... 2024-09-20T23:44:00.099 INFO:teuthology.run_tasks:Running task selinux... 2024-09-20T23:44:00.108 DEBUG:teuthology.orchestra.run.smithi018:> sudo service auditd rotate 2024-09-20T23:44:00.152 INFO:teuthology.orchestra.run.smithi018.stdout:Rotating logs: 2024-09-20T23:44:00.154 DEBUG:teuthology.orchestra.run.smithi031:> sudo service auditd rotate 2024-09-20T23:44:00.213 INFO:teuthology.orchestra.run.smithi031.stdout:Rotating logs: 2024-09-20T23:44:00.215 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-20T23:44:00.215 DEBUG:teuthology.orchestra.run.smithi018:> /usr/sbin/getenforce 2024-09-20T23:44:00.239 INFO:teuthology.orchestra.run.smithi018.stdout:Permissive 2024-09-20T23:44:00.239 DEBUG:teuthology.orchestra.run.smithi031:> /usr/sbin/getenforce 2024-09-20T23:44:00.274 INFO:teuthology.orchestra.run.smithi031.stdout:Permissive 2024-09-20T23:44:00.274 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi018.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi031.front.sepia.ceph.com': 'permissive'} 2024-09-20T23:44:00.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-20T23:44:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T23:44:00.309 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-20T23:44:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T23:44:00.343 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-20T23:44:00.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo /usr/sbin/setenforce permissive 2024-09-20T23:44:00.380 DEBUG:teuthology.orchestra.run.smithi031:> sudo /usr/sbin/setenforce permissive 2024-09-20T23:44:00.421 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-20T23:44:00.430 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-20T23:44:00.440 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-20T23:44:00.441 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi018.front.sepia.ceph.com,smithi031.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-20T23:47:54.768 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi018.front.sepia.ceph.com'), Remote(name='ubuntu@smithi031.front.sepia.ceph.com')] 2024-09-20T23:47:54.769 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:47:54.770 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:47:54.851 DEBUG:teuthology.orchestra.run.smithi018:> true 2024-09-20T23:47:54.935 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-20T23:47:54.936 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:47:54.936 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T23:47:55.014 DEBUG:teuthology.orchestra.run.smithi031:> true 2024-09-20T23:47:55.113 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-09-20T23:47:55.113 INFO:teuthology.run_tasks:Running task clock... 2024-09-20T23:47:55.123 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-20T23:47:55.124 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T23:47:55.124 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T23:47:55.127 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T23:47:55.127 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T23:47:55.158 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T23:47:55.174 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T23:47:55.188 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T23:47:55.202 INFO:teuthology.orchestra.run.smithi018.stderr:sudo: ntpd: command not found 2024-09-20T23:47:55.205 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T23:47:55.214 INFO:teuthology.orchestra.run.smithi018.stdout:506 Cannot talk to daemon 2024-09-20T23:47:55.230 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T23:47:55.232 INFO:teuthology.orchestra.run.smithi031.stderr:sudo: ntpd: command not found 2024-09-20T23:47:55.244 INFO:teuthology.orchestra.run.smithi031.stdout:506 Cannot talk to daemon 2024-09-20T23:47:55.245 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T23:47:55.260 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T23:47:55.275 INFO:teuthology.orchestra.run.smithi031.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T23:47:55.301 INFO:teuthology.orchestra.run.smithi018.stderr:bash: line 1: ntpq: command not found 2024-09-20T23:47:55.304 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T23:47:55.304 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2024-09-20T23:47:55.305 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.305 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.305 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.305 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.331 INFO:teuthology.orchestra.run.smithi031.stderr:bash: line 1: ntpq: command not found 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:=============================================================================== 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.335 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T23:47:55.335 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-20T23:47:55.429 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': '9e2ac70bb7aea67d34f0d50002344abbe7803736'} 2024-09-20T23:47:55.429 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-20T23:47:55.430 INFO:tasks.cephadm:Cluster fsid is c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:47:55.430 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-20T23:47:55.430 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.18', 'mon.c': '[v2:172.21.15.18:3301,v1:172.21.15.18:6790]', 'mon.b': '172.21.15.31'} 2024-09-20T23:47:55.430 INFO:tasks.cephadm:First mon is mon.a on smithi018 2024-09-20T23:47:55.430 INFO:tasks.cephadm:First mgr is y 2024-09-20T23:47:55.430 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-20T23:47:55.430 DEBUG:teuthology.orchestra.run.smithi018:> sudo hostname $(hostname -s) 2024-09-20T23:47:55.457 DEBUG:teuthology.orchestra.run.smithi031:> sudo hostname $(hostname -s) 2024-09-20T23:47:55.484 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-20T23:47:55.484 DEBUG:teuthology.orchestra.run.smithi018:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.721 INFO:teuthology.orchestra.run.smithi018.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 23:47 /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.721 DEBUG:teuthology.orchestra.run.smithi031:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.849 INFO:teuthology.orchestra.run.smithi031.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 23:47 /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.850 DEBUG:teuthology.orchestra.run.smithi018:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.868 DEBUG:teuthology.orchestra.run.smithi031:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T23:47:55.896 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-20T23:47:55.896 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T23:47:55.911 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T23:47:57.503 INFO:teuthology.orchestra.run.smithi031.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T23:47:57.608 INFO:teuthology.orchestra.run.smithi018.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T23:48:10.143 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:48:10.143 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout: "repo_digests": [ 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-20T23:48:10.144 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:48:10.207 INFO:teuthology.orchestra.run.smithi031.stdout:{ 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: "repo_digests": [ 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout: ] 2024-09-20T23:48:10.208 INFO:teuthology.orchestra.run.smithi031.stdout:} 2024-09-20T23:48:10.226 DEBUG:teuthology.orchestra.run.smithi018:> sudo mkdir -p /etc/ceph 2024-09-20T23:48:10.255 DEBUG:teuthology.orchestra.run.smithi031:> sudo mkdir -p /etc/ceph 2024-09-20T23:48:10.285 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 777 /etc/ceph 2024-09-20T23:48:10.321 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 777 /etc/ceph 2024-09-20T23:48:10.351 INFO:tasks.cephadm:Writing seed config... 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-20T23:48:10.352 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-20T23:48:10.353 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-20T23:48:10.353 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:48:10.353 DEBUG:teuthology.orchestra.run.smithi018:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-20T23:48:10.377 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = c1f9e122-77aa-11ef-baf5-efdc52797490 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-20T23:48:10.377 DEBUG:teuthology.orchestra.run.smithi018:mon.a> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service 2024-09-20T23:48:10.421 DEBUG:teuthology.orchestra.run.smithi018:mgr.y> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service 2024-09-20T23:48:10.464 INFO:tasks.cephadm:Bootstrapping... 2024-09-20T23:48:10.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.18 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-20T23:48:10.660 INFO:teuthology.orchestra.run.smithi018.stderr:-------------------------------------------------------------------------------- 2024-09-20T23:48:10.660 INFO:teuthology.orchestra.run.smithi018.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'c1f9e122-77aa-11ef-baf5-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.18', '--skip-admin-label'] 2024-09-20T23:48:10.683 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-20T23:48:10.685 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying podman|docker is present... 2024-09-20T23:48:10.705 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-20T23:48:10.708 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying lvm2 is present... 2024-09-20T23:48:10.708 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying time synchronization is in place... 2024-09-20T23:48:10.715 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T23:48:10.722 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:48:10.729 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: enabled 2024-09-20T23:48:10.736 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: active 2024-09-20T23:48:10.736 INFO:teuthology.orchestra.run.smithi018.stderr:Unit chronyd.service is enabled and running 2024-09-20T23:48:10.736 INFO:teuthology.orchestra.run.smithi018.stderr:Repeating the final host check... 2024-09-20T23:48:10.757 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-20T23:48:10.759 INFO:teuthology.orchestra.run.smithi018.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-20T23:48:10.759 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl is present 2024-09-20T23:48:10.759 INFO:teuthology.orchestra.run.smithi018.stderr:lvcreate is present 2024-09-20T23:48:10.765 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T23:48:10.772 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:48:10.779 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: enabled 2024-09-20T23:48:10.785 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: active 2024-09-20T23:48:10.786 INFO:teuthology.orchestra.run.smithi018.stderr:Unit chronyd.service is enabled and running 2024-09-20T23:48:10.786 INFO:teuthology.orchestra.run.smithi018.stderr:Host looks OK 2024-09-20T23:48:10.786 INFO:teuthology.orchestra.run.smithi018.stderr:Cluster fsid: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:10.786 INFO:teuthology.orchestra.run.smithi018.stderr:Acquiring lock 140361319572096 on /run/cephadm/c1f9e122-77aa-11ef-baf5-efdc52797490.lock 2024-09-20T23:48:10.786 INFO:teuthology.orchestra.run.smithi018.stderr:Lock 140361319572096 acquired on /run/cephadm/c1f9e122-77aa-11ef-baf5-efdc52797490.lock 2024-09-20T23:48:10.787 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying IP 172.21.15.18 port 3300 ... 2024-09-20T23:48:10.787 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying IP 172.21.15.18 port 6789 ... 2024-09-20T23:48:10.787 INFO:teuthology.orchestra.run.smithi018.stderr:Base mon IP is 172.21.15.18, final addrv is [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2024-09-20T23:48:10.789 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.18 metric 100 2024-09-20T23:48:10.789 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.18 metric 100 2024-09-20T23:48:10.792 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-20T23:48:10.792 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-20T23:48:10.793 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1780sec hoplimit 64 pref medium 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:1362/64 scope link 2024-09-20T23:48:10.795 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T23:48:10.796 INFO:teuthology.orchestra.run.smithi018.stderr:Mon IP `172.21.15.18` is in CIDR network `172.21.0.0/20` 2024-09-20T23:48:10.796 INFO:teuthology.orchestra.run.smithi018.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-20T23:48:10.797 INFO:teuthology.orchestra.run.smithi018.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T23:48:10.899 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-20T23:48:11.241 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Getting image source signatures 2024-09-20T23:48:11.241 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-20T23:48:11.241 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-20T23:48:11.241 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-20T23:48:11.241 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-20T23:48:11.242 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-20T23:48:11.242 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T23:48:11.421 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Writing manifest to image destination 2024-09-20T23:48:11.574 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T23:48:12.016 INFO:teuthology.orchestra.run.smithi018.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T23:48:12.516 INFO:teuthology.orchestra.run.smithi018.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T23:48:12.517 INFO:teuthology.orchestra.run.smithi018.stderr:Extracting ceph user uid/gid from container image... 2024-09-20T23:48:13.894 INFO:teuthology.orchestra.run.smithi018.stderr:stat: 167 167 2024-09-20T23:48:19.369 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial keys... 2024-09-20T23:48:22.942 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQDGCe5m6WGJChAAjHu2nvvDlB7Hy/UR/VaBhA== 2024-09-20T23:48:24.049 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQDHCe5mShj3ORAAUg5ooagJaxRcr4yyLInXiw== 2024-09-20T23:48:24.954 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQDICe5miKphNBAA4pWSCx0f7KzoPc92pZDJvA== 2024-09-20T23:48:25.437 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial monmap... 2024-09-20T23:48:25.835 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T23:48:25.835 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-20T23:48:25.835 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:25.835 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T23:48:26.475 INFO:teuthology.orchestra.run.smithi018.stderr:monmaptool for a [v2:172.21.15.18:3300,v1:172.21.15.18:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T23:48:26.476 INFO:teuthology.orchestra.run.smithi018.stderr:setting min_mon_release = octopus 2024-09-20T23:48:26.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: set fsid to c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:26.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T23:48:26.476 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:48:26.476 INFO:teuthology.orchestra.run.smithi018.stderr:Creating mon... 2024-09-20T23:48:29.338 INFO:teuthology.orchestra.run.smithi018.stderr:create mon.a on 2024-09-20T23:48:29.584 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-20T23:48:29.741 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target. 2024-09-20T23:48:29.741 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target. 2024-09-20T23:48:30.138 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to reset failed state of unit ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service: Unit ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service not loaded. 2024-09-20T23:48:30.146 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target.wants/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service → /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@.service. 2024-09-20T23:48:30.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 podman[23384]: 2024-09-20 23:48:30.570057011 +0000 UTC m=+0.128392784 container create acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD) 2024-09-20T23:48:30.870 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-20T23:48:30.877 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:48:30.878 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-20T23:48:30.878 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T23:48:30.879 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mon to start... 2024-09-20T23:48:30.879 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mon... 2024-09-20T23:48:30.999 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 podman[23384]: 2024-09-20 23:48:30.772266993 +0000 UTC m=+0.330602771 container init acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.expose-services=, name=centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-type=git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc.) 2024-09-20T23:48:30.999 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 podman[23384]: 2024-09-20 23:48:30.775711165 +0000 UTC m=+0.334046933 container start acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=) 2024-09-20T23:48:30.999 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 ceph-mon[23403]: mkfs c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:31.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 ceph-mon[23403]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-20T23:48:31.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 bash[23384]: acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 2024-09-20T23:48:31.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:30 smithi018 systemd[1]: Started Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:48:31.383 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: cluster: 2024-09-20T23:48:31.383 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: id: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: services: 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.568279s) 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: data: 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-20T23:48:31.384 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-20T23:48:31.385 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: pgs: 2024-09-20T23:48:31.385 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:31.917 INFO:teuthology.orchestra.run.smithi018.stderr:mon is available 2024-09-20T23:48:31.917 INFO:teuthology.orchestra.run.smithi018.stderr:Assimilating anything we can from ceph.conf... 2024-09-20T23:48:31.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-20T23:48:31.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: monmap e1: 1 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} 2024-09-20T23:48:31.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: fsmap 2024-09-20T23:48:31.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T23:48:31.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: mgrmap e1: no daemons active 2024-09-20T23:48:31.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:31 smithi018 ceph-mon[23403]: from='client.? 172.21.15.18:0/3670731213' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [global] 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: fsid = c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:32.410 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [mgr] 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [osd] 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T23:48:32.411 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T23:48:32.938 INFO:teuthology.orchestra.run.smithi018.stderr:Generating new minimal ceph.conf... 2024-09-20T23:48:33.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:32 smithi018 ceph-mon[23403]: from='client.? 172.21.15.18:0/2076679576' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T23:48:33.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:32 smithi018 ceph-mon[23403]: from='client.? 172.21.15.18:0/2076679576' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-20T23:48:33.987 INFO:teuthology.orchestra.run.smithi018.stderr:Restarting the monitor... 2024-09-20T23:48:34.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:33 smithi018 ceph-mon[23403]: from='client.? 172.21.15.18:0/319238483' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:48:34.421 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 systemd[1]: Stopping Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:48:34.421 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 bash[23622]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon.a" found: no such container 2024-09-20T23:48:34.421 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[23399]: 2024-09-20T23:48:34.153+0000 7fe044f99700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T23:48:34.421 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[23399]: 2024-09-20T23:48:34.153+0000 7fe044f99700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-20T23:48:34.422 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 podman[23633]: 2024-09-20 23:48:34.164259118 +0000 UTC m=+0.113279551 container died acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, maintainer=Guillaume Abrioux , ceph=True, build-date=2022-05-03T08:36:31.336870) 2024-09-20T23:48:34.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 podman[23633]: 2024-09-20 23:48:34.519428033 +0000 UTC m=+0.468448458 container cleanup acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, RELEASE=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-09-20T23:48:34.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 bash[23633]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a 2024-09-20T23:48:34.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 bash[23658]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon.a" found: no such container 2024-09-20T23:48:35.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 podman[23645]: 2024-09-20 23:48:34.778656049 +0000 UTC m=+0.611639399 container remove acc0f309cd3b9c6d4a7afb290d920b06f3dcd12bcd45d0efbe7e27f4fca80038 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, release=754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0) 2024-09-20T23:48:35.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-20T23:48:35.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 systemd[1]: Stopped Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:48:35.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:34 smithi018 systemd[1]: Starting Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:48:35.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 podman[23722]: 2024-09-20 23:48:34.973028447 +0000 UTC m=+0.017388108 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 podman[23722]: 2024-09-20 23:48:35.071100132 +0000 UTC m=+0.115459792 container create 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, version=8, name=centos-stream, io.buildah.version=1.19.8, distribution-scope=public, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 podman[23722]: 2024-09-20 23:48:35.27357813 +0000 UTC m=+0.317937808 container init 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_CLEAN=True, version=8, io.buildah.version=1.19.8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, release=754, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc.) 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 podman[23722]: 2024-09-20 23:48:35.276960606 +0000 UTC m=+0.321320265 container start 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8) 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: pidfile_write: ignore empty --pid-file 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: load: jerasure load: lrc 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: RocksDB version: 6.15.5 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Compile date Apr 18 2022 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: DB SUMMARY 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: DB Session ID: OIFM371VDNMZIRS0349X 2024-09-20T23:48:35.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: CURRENT file: CURRENT 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82633 ; 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.error_if_exists: 0 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.create_if_missing: 0 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.paranoid_checks: 1 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.env: 0x5643c2cc6860 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.fs: Posix File System 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.info_log: 0x5643c485bdc0 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.statistics: (nil) 2024-09-20T23:48:35.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.use_fsync: 0 2024-09-20T23:48:35.366 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_log_file_size: 0 2024-09-20T23:48:35.366 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T23:48:35.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T23:48:35.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T23:48:35.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T23:48:35.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_fallocate: 1 2024-09-20T23:48:35.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.use_direct_reads: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.db_log_dir: 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T23:48:35.368 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.write_buffer_manager: 0x5643c494c2a0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.unordered_write: 0 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T23:48:35.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.row_cache: None 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.wal_filter: None 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.preserve_deletes: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.two_write_queues: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.atomic_flush: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.log_readahead_size: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T23:48:35.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_background_jobs: 2 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_background_compactions: -1 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_subcompactions: 1 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T23:48:35.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_open_files: -1 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_background_flushes: -1 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Compression algorithms supported: 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kZSTD supported: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kXpressCompression supported: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kLZ4Compression supported: 1 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kBZip2Compression supported: 0 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kZlibCompression supported: 1 2024-09-20T23:48:35.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: kSnappyCompression supported: 1 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.merge_operator: 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_filter: None 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5643c4827d00) 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks: 1 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T23:48:35.373 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_top_level_index_and_filter: 1 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_type: 0 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_index_type: 0 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_shortening: 1 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: hash_index_allow_collision: 1 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: checksum: 1 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: no_block_cache: 0 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache: 0x5643c4892170 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_name: BinnedLRUCache 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_options: 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: capacity : 536870912 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_shard_bits : 4 2024-09-20T23:48:35.374 INFO:journalctl@ceph.mon.a.smithi018.stdout: strict_capacity_limit : 0 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: high_pri_pool_ratio: 0.000 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_compressed: (nil) 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: persistent_cache: (nil) 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size: 4096 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size_deviation: 10 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_restart_interval: 16 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_block_restart_interval: 1 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: metadata_block_size: 4096 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: partition_filters: 0 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: use_delta_encoding: 1 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: whole_key_filtering: 1 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: verify_compression: 0 2024-09-20T23:48:35.375 INFO:journalctl@ceph.mon.a.smithi018.stdout: read_amp_bytes_per_bit: 0 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout: format_version: 4 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_index_compression: 1 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_align: 0 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression: NoCompression 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.num_levels: 7 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T23:48:35.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T23:48:35.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T23:48:35.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T23:48:35.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T23:48:35.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T23:48:35.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T23:48:35.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T23:48:35.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.arena_block_size: 4194304 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T23:48:35.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.table_properties_collectors: 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.inplace_update_support: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.bloom_locality: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.max_successive_merges: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T23:48:35.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.ttl: 2592000 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.enable_blob_files: false 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.min_blob_size: 0 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-20T23:48:35.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876115304320, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876115305109, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79183, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77446, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71755, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726876115, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a4fa1e9f-3b76-4262-bd89-0375c0435f77", "db_session_id": "OIFM371VDNMZIRS0349X"}} 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876115305463, "job": 1, "event": "recovery_finished"} 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5643c4878a80 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: DB pointer 0x5643c4888000 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** DB Stats ** 2024-09-20T23:48:35.383 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.384 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 23.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 23.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:48:35.385 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:48:35.386 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 146.6 0.00 0.00 1 0.001 0 0 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 22.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: starting mon.a rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:35.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???) e1 preinit fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).mds e1 new map 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).mds e1 print_map 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: e1 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: legacy client fscid: -1 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout: No filesystems configured 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expand map: {default=false} 2024-09-20T23:48:35.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta from 'false' to 'false' 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expanded map: {default=false} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expand map: {default=info} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta from 'info' to 'info' 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expanded map: {default=info} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expand map: {default=daemon} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expanded map: {default=daemon} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expand map: {default=debug} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta from 'debug' to 'debug' 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: expand_channel_meta expanded map: {default=debug} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: monmap e1: 1 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: fsmap 2024-09-20T23:48:35.389 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T23:48:35.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 ceph-mon[23740]: mgrmap e1: no daemons active 2024-09-20T23:48:35.395 INFO:teuthology.orchestra.run.smithi018.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-20T23:48:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 bash[23722]: 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda 2024-09-20T23:48:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:35 smithi018 systemd[1]: Started Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:48:36.435 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-20T23:48:36.435 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-20T23:48:36.435 INFO:teuthology.orchestra.run.smithi018.stderr:Creating mgr... 2024-09-20T23:48:36.435 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying port 9283 ... 2024-09-20T23:48:36.619 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to reset failed state of unit ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service: Unit ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service not loaded. 2024-09-20T23:48:36.627 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490.target.wants/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@.service. 2024-09-20T23:48:37.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:36 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1647837636' entity='client.admin' 2024-09-20T23:48:37.330 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-20T23:48:37.338 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:48:37.339 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-20T23:48:37.339 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T23:48:37.348 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-20T23:48:37.357 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:48:37.357 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-20T23:48:37.357 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-20T23:48:37.358 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr to start... 2024-09-20T23:48:37.358 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr... 2024-09-20T23:48:37.756 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:37.754+0000 7f5b0c866000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:48:37.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:37.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:37.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "c1f9e122-77aa-11ef-baf5-efdc52797490", 2024-09-20T23:48:37.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T23:48:37.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "a" 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T23:48:37.958 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T23:48:37.959 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T23:48:37.960 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T23:48:37.960 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T23:48:37.960 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T23:48:37.960 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T23:48:37.960 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T23:48:37.961 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T23:48:37.962 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": false, 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-20T23:48:37.963 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-20T23:48:30.813023+0000", 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T23:48:37.964 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:38.117 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:37 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2318434321' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T23:48:38.117 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:37.894+0000 7f5b0c866000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:48:38.117 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:38.115+0000 7f5b0c866000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:48:38.723 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:38.459+0000 7f5b0c866000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:48:38.723 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:38.550+0000 7f5b0c866000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:48:38.724 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:38.722+0000 7f5b0c866000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:48:38.827 INFO:teuthology.orchestra.run.smithi018.stderr:mgr not available, waiting (1/15)... 2024-09-20T23:48:40.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:39.814+0000 7f5b0c866000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:48:40.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:39.907+0000 7f5b0c866000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:48:41.172 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:40.852+0000 7f5b0c866000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "c1f9e122-77aa-11ef-baf5-efdc52797490", 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T23:48:41.396 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "a" 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T23:48:41.397 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T23:48:41.400 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T23:48:41.400 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.401 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": false, 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T23:48:41.402 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:41.403 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-20T23:48:30.813023+0000", 2024-09-20T23:48:41.403 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:41.403 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:41.403 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T23:48:41.403 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:41.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.170+0000 7f5b0c866000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:48:41.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.311+0000 7f5b0c866000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:48:41.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.406+0000 7f5b0c866000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:48:41.756 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2099294101' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T23:48:41.756 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.510+0000 7f5b0c866000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:48:41.756 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.600+0000 7f5b0c866000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:48:41.756 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.754+0000 7f5b0c866000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:48:41.977 INFO:teuthology.orchestra.run.smithi018.stderr:mgr not available, waiting (2/15)... 2024-09-20T23:48:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.849+0000 7f5b0c866000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:48:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:41.948+0000 7f5b0c866000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:48:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:42.241+0000 7f5b0c866000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:48:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:42.330+0000 7f5b0c866000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:48:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:42.420+0000 7f5b0c866000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:48:43.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:43.526+0000 7f5b0c866000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:48:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: Activating manager daemon y 2024-09-20T23:48:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: mgrmap e2: y(active, starting, since 0.00284088s) 2024-09-20T23:48:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:48:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' 2024-09-20T23:48:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:43 smithi018 ceph-mon[23740]: from='mgr.14100 172.21.15.18:0/3727114364' entity='mgr.y' 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "c1f9e122-77aa-11ef-baf5-efdc52797490", 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T23:48:44.471 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "a" 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T23:48:44.472 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T23:48:44.473 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T23:48:44.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T23:48:44.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T23:48:44.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.476 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T23:48:44.477 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": false, 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-20T23:48:30.813023+0000", 2024-09-20T23:48:44.478 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:44.479 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:44.479 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T23:48:44.479 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:44.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:44 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3658811314' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T23:48:44.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:44 smithi018 ceph-mon[23740]: mgrmap e3: y(active, since 1.00587s) 2024-09-20T23:48:45.031 INFO:teuthology.orchestra.run.smithi018.stderr:mgr not available, waiting (3/15)... 2024-09-20T23:48:47.422 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:47 smithi018 ceph-mon[23740]: mgrmap e4: y(active, since 2s) 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "c1f9e122-77aa-11ef-baf5-efdc52797490", 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T23:48:47.721 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "a" 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:47.722 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T23:48:47.723 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T23:48:47.726 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T23:48:47.727 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-20T23:48:47.728 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-20T23:48:30.813023+0000", 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T23:48:47.729 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:48.262 INFO:teuthology.orchestra.run.smithi018.stderr:mgr is available 2024-09-20T23:48:48.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2976813167' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [global] 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: fsid = c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:48.867 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [mgr] 2024-09-20T23:48:48.868 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T23:48:48.868 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-20T23:48:48.868 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [osd] 2024-09-20T23:48:48.868 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T23:48:48.868 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T23:48:49.343 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling cephadm module... 2024-09-20T23:48:49.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:49 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1991746647' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T23:48:49.566 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:49 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1991746647' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-20T23:48:50.478 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1895216013' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-20T23:48:50.478 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setuser ceph since I am not root 2024-09-20T23:48:50.478 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setgroup ceph since I am not root 2024-09-20T23:48:50.478 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:50.476+0000 7f5a9d992000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-20T23:48:50.752 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:50.558+0000 7f5a9d992000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:48:50.752 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:50.750+0000 7f5a9d992000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:48:51.047 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:50.842+0000 7f5a9d992000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:48:51.047 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:50.981+0000 7f5a9d992000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:48:51.318 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:51.202+0000 7f5a9d992000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:48:51.579 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:51.553+0000 7f5a9d992000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:48:51.579 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1895216013' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-20T23:48:51.579 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-mon[23740]: mgrmap e5: y(active, since 6s) 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T23:48:51.580 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:51.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:51.648+0000 7f5a9d992000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:48:51.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:51.820+0000 7f5a9d992000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:48:52.189 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for the mgr to restart... 2024-09-20T23:48:52.189 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr epoch 5... 2024-09-20T23:48:52.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:52 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2671419112' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T23:48:53.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:52.927+0000 7f5a9d992000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:48:53.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:53.021+0000 7f5a9d992000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:48:54.309 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:53.967+0000 7f5a9d992000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:48:54.309 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.307+0000 7f5a9d992000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:48:54.643 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.446+0000 7f5a9d992000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:48:54.643 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.538+0000 7f5a9d992000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:48:54.643 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.641+0000 7f5a9d992000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:48:54.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.732+0000 7f5a9d992000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:48:54.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.886+0000 7f5a9d992000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:48:55.378 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:54.983+0000 7f5a9d992000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:48:55.379 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:55.083+0000 7f5a9d992000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:48:55.379 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:55.377+0000 7f5a9d992000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:48:55.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:55.468+0000 7f5a9d992000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:48:55.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:55.560+0000 7f5a9d992000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:48:56.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:48:56.669+0000 7f5a9d992000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:48:57.734 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:48:57.735 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-20T23:48:57.735 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T23:48:57.735 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: Active manager daemon y restarted 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: Activating manager daemon y 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: osdmap e2: 0 total, 0 up, 0 in 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: mgrmap e6: y(active, starting, since 0.0526929s) 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:48:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:48:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:57 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:48:58.303 INFO:teuthology.orchestra.run.smithi018.stderr:mgr epoch 5 is available 2024-09-20T23:48:58.303 INFO:teuthology.orchestra.run.smithi018.stderr:Setting orchestrator backend to cephadm... 2024-09-20T23:48:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:58 smithi018 ceph-mon[23740]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T23:48:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:58 smithi018 ceph-mon[23740]: mgrmap e7: y(active, since 1.05515s) 2024-09-20T23:48:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:58 smithi018 ceph-mon[23740]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T23:48:59.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:48:58] ENGINE Bus STARTING 2024-09-20T23:48:59.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:48:59] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:48:59.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:48:59] ENGINE Bus STARTED 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: mgrmap e8: y(active, since 2s) 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: [20/Sep/2024:23:48:58] ENGINE Bus STARTING 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: [20/Sep/2024:23:48:59] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: [20/Sep/2024:23:48:59] ENGINE Bus STARTED 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:00.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:48:59 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:00.050 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: value unchanged 2024-09-20T23:49:00.573 INFO:teuthology.orchestra.run.smithi018.stderr:Generating ssh key... 2024-09-20T23:49:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-mon[23740]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:01.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: Generating public/private rsa key pair. 2024-09-20T23:49:01.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: Your identification has been saved in /tmp/tmpgmszjv5c/key. 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: Your public key has been saved in /tmp/tmpgmszjv5c/key.pub. 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: The key fingerprint is: 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: SHA256:r6fProhBjR7HmqnAXHiFr0wCiBkC+UK43JVnRoRX6CE ceph-c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: The key's randomart image is: 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: +---[RSA 3072]----+ 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |*. =oo. | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |*+ .E B | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |Bo....B . | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |+.o.o+ . | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: | + ++.+ S | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |o *o.* . | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: |.o o* . | 2024-09-20T23:49:01.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: | . . o . o. | 2024-09-20T23:49:01.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: | . . . +*+ | 2024-09-20T23:49:01.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: +----[SHA256]-----+ 2024-09-20T23:49:02.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:02 smithi018 ceph-mon[23740]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:02.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:02 smithi018 ceph-mon[23740]: Generating ssh key... 2024-09-20T23:49:02.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:02 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:02.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:02 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:02.602 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClPA23BLN6V/aDCtLVOjdaYrxn38qIx0QIN2yVR5+6N8m9JH2iFpsDUtOwE6WiaI4hHNbhYNNTWEt9yEu61EBs2h+yu2RdUhnaSOAqhbaI3OHyF9khP5tdEPzhfeafDnx/WgV5Kc8fClzDgtXvb83DL+o//oqR/L83mLpr/fiQHMzh4wH7ZRX5wppIa81jl0+r/WdHDT7VJ8BoS+se7vt+zM5CMz4+rnvZzo2uTwafz2ozP67+lU4wg/LDz3kpFloj0PIdDy0Z0E0m1glzEEfr0JRn2iRhrqUoJBLwFL4et/+JOQdoZ/9FZx9AO1hhZ7Wz6VBVxU+mr5mbjNh9iC2kNsOQQ4GqxQ8Vtt2BnjJzUDdCu+hhsXeUv+AhWVFbjZfGEuqIl/J21dCf3ZhZft/bW8FtxzVWjmLV8ipRkE1DbVhqrLwYpQsLKQowK5+O7QbjeqC3RDvnQ/oBnSDL6IdsUjJvodrC55Ly86T3zDuK1wa6zbsziNNn3QRlLVicSPM= ceph-c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:03.204 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-20T23:49:03.204 INFO:teuthology.orchestra.run.smithi018.stderr:Adding key to root@localhost authorized_keys... 2024-09-20T23:49:03.207 INFO:teuthology.orchestra.run.smithi018.stderr:Adding host smithi018... 2024-09-20T23:49:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:03 smithi018 ceph-mon[23740]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:04 smithi018 ceph-mon[23740]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi018", "addr": "172.21.15.18", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:04 smithi018 ceph-mon[23740]: Deploying cephadm binary to smithi018 2024-09-20T23:49:04.736 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Added host 'smithi018' with addr '172.21.15.18' 2024-09-20T23:49:05.312 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying unmanaged mon service... 2024-09-20T23:49:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:05 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:05 smithi018 ceph-mon[23740]: Added host smithi018 2024-09-20T23:49:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:05 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:06.018 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-20T23:49:06.736 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying unmanaged mgr service... 2024-09-20T23:49:07.321 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:07 smithi018 ceph-mon[23740]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:07.321 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:07 smithi018 ceph-mon[23740]: Saving service mon spec with placement count:5 2024-09-20T23:49:07.321 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:07 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:07.426 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-20T23:49:08.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:08 smithi018 ceph-mon[23740]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:08.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:08 smithi018 ceph-mon[23740]: Saving service mgr spec with placement count:2 2024-09-20T23:49:08.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:08 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:09 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:09 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:09 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3315202736' entity='client.admin' 2024-09-20T23:49:11.168 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling the dashboard module... 2024-09-20T23:49:11.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:11 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1098610466' entity='client.admin' 2024-09-20T23:49:12.863 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1255441094' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-20T23:49:12.863 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:12.863 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-mon[23740]: from='mgr.14120 172.21.15.18:0/1628322056' entity='mgr.y' 2024-09-20T23:49:12.863 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setuser ceph since I am not root 2024-09-20T23:49:12.864 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setgroup ceph since I am not root 2024-09-20T23:49:12.864 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:12.779+0000 7ff366039000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-20T23:49:12.864 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:12.861+0000 7ff366039000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:49:13.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.053+0000 7ff366039000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:49:13.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.145+0000 7ff366039000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:49:13.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.289+0000 7ff366039000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:49:13.795 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1255441094' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-20T23:49:13.795 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-mon[23740]: mgrmap e9: y(active, since 15s) 2024-09-20T23:49:13.796 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.511+0000 7ff366039000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:49:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:49:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-20T23:49:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-20T23:49:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-20T23:49:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T23:49:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:49:14.111 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.869+0000 7ff366039000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:49:14.112 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:13.960+0000 7ff366039000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:49:14.387 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:14.132+0000 7ff366039000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:49:14.390 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for the mgr to restart... 2024-09-20T23:49:14.390 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr epoch 9... 2024-09-20T23:49:14.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:14 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2905998201' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T23:49:15.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:15.237+0000 7ff366039000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:49:15.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:15.331+0000 7ff366039000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:49:16.607 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:16.268+0000 7ff366039000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:49:16.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:16.605+0000 7ff366039000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:49:16.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:16.744+0000 7ff366039000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:49:16.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:16.836+0000 7ff366039000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:49:17.281 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:16.938+0000 7ff366039000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:49:17.281 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.029+0000 7ff366039000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:49:17.281 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.182+0000 7ff366039000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:49:17.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.279+0000 7ff366039000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:49:17.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.380+0000 7ff366039000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:49:18.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.675+0000 7ff366039000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:49:18.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.767+0000 7ff366039000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:49:18.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:17.859+0000 7ff366039000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:49:19.227 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:49:18.973+0000 7ff366039000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:49:19.741 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:49:19] ENGINE Bus STARTING 2024-09-20T23:49:19.742 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:49:19] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:49:19.742 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:49:19] ENGINE Bus STARTED 2024-09-20T23:49:20.036 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: Active manager daemon y restarted 2024-09-20T23:49:20.036 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: Activating manager daemon y 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: osdmap e3: 0 total, 0 up, 0 in 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: mgrmap e10: y(active, starting, since 0.0525862s) 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:49:20.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: [20/Sep/2024:23:49:19] ENGINE Bus STARTING 2024-09-20T23:49:20.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: [20/Sep/2024:23:49:19] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:49:20.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: [20/Sep/2024:23:49:19] ENGINE Bus STARTED 2024-09-20T23:49:20.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:20.040 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-20T23:49:20.040 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-20T23:49:20.040 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T23:49:20.041 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-20T23:49:20.666 INFO:teuthology.orchestra.run.smithi018.stderr:mgr epoch 9 is available 2024-09-20T23:49:20.666 INFO:teuthology.orchestra.run.smithi018.stderr:Generating a dashboard self-signed certificate... 2024-09-20T23:49:21.407 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:21 smithi018 ceph-mon[23740]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T23:49:21.407 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:21 smithi018 ceph-mon[23740]: mgrmap e11: y(active, since 1.05635s) 2024-09-20T23:49:21.407 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:21 smithi018 ceph-mon[23740]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T23:49:21.407 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:21.560 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-20T23:49:22.230 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:22.230 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:22 smithi018 ceph-mon[23740]: mgrmap e12: y(active, since 2s) 2024-09-20T23:49:22.231 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:22 smithi018 ceph-mon[23740]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:22.231 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:22.231 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:22.844 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial admin user... 2024-09-20T23:49:23.753 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$ohrgc5HnBux48r.0n5S4Q.FuZkZ5qrhZV/961bg6p6jWm51q7Jy3q", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726876163, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-20T23:49:24.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:23 smithi018 ceph-mon[23740]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:24.342 INFO:teuthology.orchestra.run.smithi018.stderr:Fetching dashboard port number... 2024-09-20T23:49:24.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:24.999 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 8443 2024-09-20T23:49:25.608 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-20T23:49:25.616 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-20T23:49:25.616 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-20T23:49:25.616 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr:Ceph Dashboard is now available at: 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr: URL: https://smithi018.front.sepia.ceph.com:8443/ 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr: User: admin 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr: Password: ol922uccze 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:25.619 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling autotune for osd_memory_target 2024-09-20T23:49:25.872 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:25 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2060169308' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-20T23:49:26.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:26 smithi018 ceph-mon[23740]: mgrmap e13: y(active, since 6s) 2024-09-20T23:49:27.692 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr:You can access the Ceph CLI with: 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-20T23:49:28.269 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr: ceph telemetry on 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr:For more information see: 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-20T23:49:28.270 INFO:teuthology.orchestra.run.smithi018.stderr:Bootstrap complete. 2024-09-20T23:49:28.304 INFO:tasks.cephadm:Fetching config... 2024-09-20T23:49:28.305 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:49:28.305 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-20T23:49:28.322 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-20T23:49:28.322 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:49:28.322 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-20T23:49:28.378 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-20T23:49:28.379 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:49:28.379 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-20T23:49:28.444 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-20T23:49:28.444 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:49:28.444 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-20T23:49:28.500 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-20T23:49:28.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClPA23BLN6V/aDCtLVOjdaYrxn38qIx0QIN2yVR5+6N8m9JH2iFpsDUtOwE6WiaI4hHNbhYNNTWEt9yEu61EBs2h+yu2RdUhnaSOAqhbaI3OHyF9khP5tdEPzhfeafDnx/WgV5Kc8fClzDgtXvb83DL+o//oqR/L83mLpr/fiQHMzh4wH7ZRX5wppIa81jl0+r/WdHDT7VJ8BoS+se7vt+zM5CMz4+rnvZzo2uTwafz2ozP67+lU4wg/LDz3kpFloj0PIdDy0Z0E0m1glzEEfr0JRn2iRhrqUoJBLwFL4et/+JOQdoZ/9FZx9AO1hhZ7Wz6VBVxU+mr5mbjNh9iC2kNsOQQ4GqxQ8Vtt2BnjJzUDdCu+hhsXeUv+AhWVFbjZfGEuqIl/J21dCf3ZhZft/bW8FtxzVWjmLV8ipRkE1DbVhqrLwYpQsLKQowK5+O7QbjeqC3RDvnQ/oBnSDL6IdsUjJvodrC55Ly86T3zDuK1wa6zbsziNNn3QRlLVicSPM= ceph-c1f9e122-77aa-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T23:49:28.579 INFO:teuthology.orchestra.run.smithi018.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClPA23BLN6V/aDCtLVOjdaYrxn38qIx0QIN2yVR5+6N8m9JH2iFpsDUtOwE6WiaI4hHNbhYNNTWEt9yEu61EBs2h+yu2RdUhnaSOAqhbaI3OHyF9khP5tdEPzhfeafDnx/WgV5Kc8fClzDgtXvb83DL+o//oqR/L83mLpr/fiQHMzh4wH7ZRX5wppIa81jl0+r/WdHDT7VJ8BoS+se7vt+zM5CMz4+rnvZzo2uTwafz2ozP67+lU4wg/LDz3kpFloj0PIdDy0Z0E0m1glzEEfr0JRn2iRhrqUoJBLwFL4et/+JOQdoZ/9FZx9AO1hhZ7Wz6VBVxU+mr5mbjNh9iC2kNsOQQ4GqxQ8Vtt2BnjJzUDdCu+hhsXeUv+AhWVFbjZfGEuqIl/J21dCf3ZhZft/bW8FtxzVWjmLV8ipRkE1DbVhqrLwYpQsLKQowK5+O7QbjeqC3RDvnQ/oBnSDL6IdsUjJvodrC55Ly86T3zDuK1wa6zbsziNNn3QRlLVicSPM= ceph-c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:28.590 DEBUG:teuthology.orchestra.run.smithi031:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClPA23BLN6V/aDCtLVOjdaYrxn38qIx0QIN2yVR5+6N8m9JH2iFpsDUtOwE6WiaI4hHNbhYNNTWEt9yEu61EBs2h+yu2RdUhnaSOAqhbaI3OHyF9khP5tdEPzhfeafDnx/WgV5Kc8fClzDgtXvb83DL+o//oqR/L83mLpr/fiQHMzh4wH7ZRX5wppIa81jl0+r/WdHDT7VJ8BoS+se7vt+zM5CMz4+rnvZzo2uTwafz2ozP67+lU4wg/LDz3kpFloj0PIdDy0Z0E0m1glzEEfr0JRn2iRhrqUoJBLwFL4et/+JOQdoZ/9FZx9AO1hhZ7Wz6VBVxU+mr5mbjNh9iC2kNsOQQ4GqxQ8Vtt2BnjJzUDdCu+hhsXeUv+AhWVFbjZfGEuqIl/J21dCf3ZhZft/bW8FtxzVWjmLV8ipRkE1DbVhqrLwYpQsLKQowK5+O7QbjeqC3RDvnQ/oBnSDL6IdsUjJvodrC55Ly86T3zDuK1wa6zbsziNNn3QRlLVicSPM= ceph-c1f9e122-77aa-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T23:49:28.647 INFO:teuthology.orchestra.run.smithi031.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQClPA23BLN6V/aDCtLVOjdaYrxn38qIx0QIN2yVR5+6N8m9JH2iFpsDUtOwE6WiaI4hHNbhYNNTWEt9yEu61EBs2h+yu2RdUhnaSOAqhbaI3OHyF9khP5tdEPzhfeafDnx/WgV5Kc8fClzDgtXvb83DL+o//oqR/L83mLpr/fiQHMzh4wH7ZRX5wppIa81jl0+r/WdHDT7VJ8BoS+se7vt+zM5CMz4+rnvZzo2uTwafz2ozP67+lU4wg/LDz3kpFloj0PIdDy0Z0E0m1glzEEfr0JRn2iRhrqUoJBLwFL4et/+JOQdoZ/9FZx9AO1hhZ7Wz6VBVxU+mr5mbjNh9iC2kNsOQQ4GqxQ8Vtt2BnjJzUDdCu+hhsXeUv+AhWVFbjZfGEuqIl/J21dCf3ZhZft/bW8FtxzVWjmLV8ipRkE1DbVhqrLwYpQsLKQowK5+O7QbjeqC3RDvnQ/oBnSDL6IdsUjJvodrC55Ly86T3zDuK1wa6zbsziNNn3QRlLVicSPM= ceph-c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:28.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-20T23:49:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:28 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3982218866' entity='client.admin' 2024-09-20T23:49:31.681 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-20T23:49:31.682 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1919756939' entity='client.admin' 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi018", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:32.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:31 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:34.090 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi031 2024-09-20T23:49:34.090 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:49:34.090 DEBUG:teuthology.orchestra.run.smithi031:> dd of=/etc/ceph/ceph.conf 2024-09-20T23:49:34.109 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:49:34.110 DEBUG:teuthology.orchestra.run.smithi031:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:49:34.168 INFO:tasks.cephadm:Adding host smithi031 to orchestrator... 2024-09-20T23:49:34.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch host add smithi031 2024-09-20T23:49:34.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:34.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:34.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:34.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:34.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:49:34.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-20T23:49:34.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:49:34.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:34.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:36.947 INFO:teuthology.orchestra.run.smithi018.stdout:Added host 'smithi031' with addr '172.21.15.31' 2024-09-20T23:49:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:36 smithi018 ceph-mon[23740]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi031", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:36 smithi018 ceph-mon[23740]: Deploying cephadm binary to smithi031 2024-09-20T23:49:37.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch host ls --format=json 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: Added host smithi031 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:38.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:39.309 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:49:39.310 INFO:teuthology.orchestra.run.smithi018.stdout:[{"addr": "172.21.15.18", "hostname": "smithi018", "labels": [], "status": ""}, {"addr": "172.21.15.31", "hostname": "smithi031", "labels": [], "status": ""}] 2024-09-20T23:49:39.590 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:39.863 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-20T23:49:39.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd crush tunables default 2024-09-20T23:49:40.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:40 smithi018 ceph-mon[23740]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:40.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:40 smithi018 ceph-mon[23740]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:49:42.519 INFO:teuthology.orchestra.run.smithi018.stderr:adjusted tunables profile to default 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi031", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:42.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1260651370' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-20T23:49:43.088 INFO:tasks.cephadm:Adding mon.a on smithi018 2024-09-20T23:49:43.088 INFO:tasks.cephadm:Adding mon.c on smithi018 2024-09-20T23:49:43.088 INFO:tasks.cephadm:Adding mon.b on smithi031 2024-09-20T23:49:43.088 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply mon '3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi031:172.21.15.31=b' 2024-09-20T23:49:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:43 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1260651370' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-20T23:49:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:43 smithi018 ceph-mon[23740]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:49:44.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:44 smithi018 ceph-mon[23740]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:44.979 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled mon update... 2024-09-20T23:49:45.762 DEBUG:teuthology.orchestra.run.smithi018:mon.c> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.c.service 2024-09-20T23:49:45.764 DEBUG:teuthology.orchestra.run.smithi031:mon.b> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.b.service 2024-09-20T23:49:45.766 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-20T23:49:45.766 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T23:49:46.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi031:172.21.15.31=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: Saving service mon spec with placement smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi031:172.21.15.31=b;count:3 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:46.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:49:46.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:46.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:45 smithi018 ceph-mon[23740]: Deploying daemon mon.c on smithi018 2024-09-20T23:49:47.917 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:49:47.917 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":1,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","modified":"2024-09-20T23:48:25.756842Z","created":"2024-09-20T23:48:25.756842Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T23:49:47.919 INFO:teuthology.orchestra.run.smithi031.stderr:dumped monmap epoch 1 2024-09-20T23:49:48.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:48 smithi018 ceph-mon[23740]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:48.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/489370638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:49:49.482 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-20T23:49:49.483 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T23:49:53.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: mon.a calling monitor election 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: mon.c calling monitor election 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:53.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: monmap e2: 2 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: fsmap 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: mgrmap e13: y(active, since 34s) 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: overall HEALTH_OK 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:49:53.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:53 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:54.426 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:49:54.426 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":2,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","modified":"2024-09-20T23:49:48.552473Z","created":"2024-09-20T23:48:25.756842Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3301","nonce":0},{"type":"v1","addr":"172.21.15.18:6790","nonce":0}]},"addr":"172.21.15.18:6790/0","public_addr":"172.21.15.18:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-20T23:49:54.428 INFO:teuthology.orchestra.run.smithi031.stderr:dumped monmap epoch 2 2024-09-20T23:49:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[23740]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[23740]: Deploying daemon mon.b on smithi031 2024-09-20T23:49:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/3531075331' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:49:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[28324]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[28324]: Deploying daemon mon.b on smithi031 2024-09-20T23:49:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/3531075331' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:49:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:49:56 smithi018 ceph-mon[23740]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:49:56 smithi018 ceph-mon[28324]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:57.125 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 systemd[1]: Starting Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:49:57.378 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 podman[24478]: 2024-09-20 23:49:57.296133975 +0000 UTC m=+0.099805530 container create c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, distribution-scope=public, version=8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T23:49:57.378 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 podman[24478]: 2024-09-20 23:49:57.213713757 +0000 UTC m=+0.017385322 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-20T23:49:57.582 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-20T23:49:57.583 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T23:49:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 podman[24478]: 2024-09-20 23:49:57.465926493 +0000 UTC m=+0.269598065 container init c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, distribution-scope=public, ceph=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 podman[24478]: 2024-09-20 23:49:57.469385165 +0000 UTC m=+0.273056723 container start c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, version=8, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870) 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: pidfile_write: ignore empty --pid-file 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: load: jerasure load: lrc 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: RocksDB version: 6.15.5 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-20T23:49:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Compile date Apr 18 2022 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: DB SUMMARY 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: DB Session ID: RJR890QO1PTE3YG91VB5 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: CURRENT file: CURRENT 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.error_if_exists: 0 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.create_if_missing: 0 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.paranoid_checks: 1 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.env: 0x55ea50ec6860 2024-09-20T23:49:57.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.fs: Posix File System 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.info_log: 0x55ea52257dc0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.statistics: (nil) 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.use_fsync: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_log_file_size: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_fallocate: 1 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T23:49:57.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.use_direct_reads: 0 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.db_log_dir: 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T23:49:57.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T23:49:57.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T23:49:57.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T23:49:57.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.write_buffer_manager: 0x55ea52348240 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.unordered_write: 0 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T23:49:57.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.row_cache: None 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.wal_filter: None 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.preserve_deletes: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.two_write_queues: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.atomic_flush: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.log_readahead_size: 0 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T23:49:57.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_background_jobs: 2 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_background_compactions: -1 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_subcompactions: 1 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T23:49:57.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T23:49:57.760 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_open_files: -1 2024-09-20T23:49:57.760 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T23:49:57.760 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T23:49:57.760 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T23:49:57.760 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T23:49:57.761 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_background_flushes: -1 2024-09-20T23:49:57.761 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Compression algorithms supported: 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kZSTD supported: 0 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kXpressCompression supported: 0 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kLZ4Compression supported: 1 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kBZip2Compression supported: 0 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kZlibCompression supported: 1 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: kSnappyCompression supported: 1 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.merge_operator: 2024-09-20T23:49:57.762 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_filter: None 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ea52223d00) 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: cache_index_and_filter_blocks: 1 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: pin_top_level_index_and_filter: 1 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_type: 0 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: data_block_index_type: 0 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_shortening: 1 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T23:49:57.763 INFO:journalctl@ceph.mon.b.smithi031.stdout: hash_index_allow_collision: 1 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: checksum: 1 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: no_block_cache: 0 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache: 0x55ea5228e170 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_name: BinnedLRUCache 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_options: 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: capacity : 536870912 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: num_shard_bits : 4 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: strict_capacity_limit : 0 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: high_pri_pool_ratio: 0.000 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_compressed: (nil) 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: persistent_cache: (nil) 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_size: 4096 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_size_deviation: 10 2024-09-20T23:49:57.764 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_restart_interval: 16 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_block_restart_interval: 1 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: metadata_block_size: 4096 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: partition_filters: 0 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: use_delta_encoding: 1 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: whole_key_filtering: 1 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: verify_compression: 0 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: read_amp_bytes_per_bit: 0 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: format_version: 4 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: enable_index_compression: 1 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_align: 0 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression: NoCompression 2024-09-20T23:49:57.765 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.num_levels: 7 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T23:49:57.766 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T23:49:57.767 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.arena_block_size: 4194304 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T23:49:57.768 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.table_properties_collectors: 2024-09-20T23:49:57.769 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.inplace_update_support: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.bloom_locality: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.max_successive_merges: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.ttl: 2592000 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.enable_blob_files: false 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.min_blob_size: 0 2024-09-20T23:49:57.770 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876197497400, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876197497706, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726876197, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d408432b-0f14-4b72-b64f-f5a3dec13046", "db_session_id": "RJR890QO1PTE3YG91VB5"}} 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726876197498025, "job": 1, "event": "recovery_finished"} 2024-09-20T23:49:57.771 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ea52274a80 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: DB pointer 0x55ea52284000 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** DB Stats ** 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-20T23:49:57.772 INFO:journalctl@ceph.mon.b.smithi031.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T23:49:57.773 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-20T23:49:57.774 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.775 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: using public_addr v2:172.21.15.31:0/0 -> [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: starting mon.b rank -1 at public addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] at bind addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(???) e0 preinit fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).mds e1 new map 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout: e1 2024-09-20T23:49:57.776 INFO:journalctl@ceph.mon.b.smithi031.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout: legacy client fscid: -1 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout: No filesystems configured 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T23:49:57.777 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.a calling monitor election 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.c calling monitor election 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: monmap e2: 2 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: fsmap 2024-09-20T23:49:57.778 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mgrmap e13: y(active, since 34s) 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: overall HEALTH_OK 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: Deploying daemon mon.b on smithi031 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/3531075331' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 ceph-mon[24497]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 bash[24478]: c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a 2024-09-20T23:49:57.779 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:49:57 smithi031 systemd[1]: Started Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: mon.a calling monitor election 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: mon.c calling monitor election 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: fsmap 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: mgrmap e13: y(active, since 43s) 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: mon.b (rank 2) addr [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] is down (out of quorum) 2024-09-20T23:50:02.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: mon.a calling monitor election 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: mon.c calling monitor election 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: fsmap 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: mgrmap e13: y(active, since 43s) 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-20T23:50:02.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: mon.b (rank 2) addr [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] is down (out of quorum) 2024-09-20T23:50:02.930 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:02.930 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:02.930 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:02.930 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:03.044 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:50:03.044 INFO:teuthology.orchestra.run.smithi031.stdout:{"epoch":3,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","modified":"2024-09-20T23:49:57.519417Z","created":"2024-09-20T23:48:25.756842Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3301","nonce":0},{"type":"v1","addr":"172.21.15.18:6790","nonce":0}]},"addr":"172.21.15.18:6790/0","public_addr":"172.21.15.18:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:3300","nonce":0},{"type":"v1","addr":"172.21.15.31:6789","nonce":0}]},"addr":"172.21.15.31:6789/0","public_addr":"172.21.15.31:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-20T23:50:03.047 INFO:teuthology.orchestra.run.smithi031.stderr:dumped monmap epoch 3 2024-09-20T23:50:03.711 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-20T23:50:03.711 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph config generate-minimal-conf 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/2943580862' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/2943580862' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:03.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:03 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: mon.a calling monitor election 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: mon.c calling monitor election 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: fsmap 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: mgrmap e13: y(active, since 43s) 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: mon.b (rank 2) addr [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] is down (out of quorum) 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:05.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/2943580862' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T23:50:05.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:05.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mon.b calling monitor election 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: Reconfiguring mon.a (unknown last config time)... 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: Reconfiguring daemon mon.a on smithi018 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mon.b calling monitor election 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mon.c calling monitor election 2024-09-20T23:50:05.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mon.a calling monitor election 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: fsmap 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: mgrmap e13: y(active, since 45s) 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: Cluster is now healthy 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[23740]: overall HEALTH_OK 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mon.b calling monitor election 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:50:05.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: Reconfiguring mon.a (unknown last config time)... 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: Reconfiguring daemon mon.a on smithi018 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mon.b calling monitor election 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mon.c calling monitor election 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mon.a calling monitor election 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: fsmap 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: mgrmap e13: y(active, since 45s) 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: Cluster is now healthy 2024-09-20T23:50:05.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:05 smithi018 ceph-mon[28324]: overall HEALTH_OK 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mon.b calling monitor election 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: Reconfiguring mon.a (unknown last config time)... 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: Reconfiguring daemon mon.a on smithi018 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mon.b calling monitor election 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mon.c calling monitor election 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mon.a calling monitor election 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: fsmap 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T23:50:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: mgrmap e13: y(active, since 45s) 2024-09-20T23:50:05.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-20T23:50:05.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: Cluster is now healthy 2024-09-20T23:50:05.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:05 smithi031 ceph-mon[24497]: overall HEALTH_OK 2024-09-20T23:50:05.878 INFO:teuthology.orchestra.run.smithi018.stdout:# minimal ceph.conf for c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:50:05.878 INFO:teuthology.orchestra.run.smithi018.stdout:[global] 2024-09-20T23:50:05.878 INFO:teuthology.orchestra.run.smithi018.stdout: fsid = c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:50:05.878 INFO:teuthology.orchestra.run.smithi018.stdout: mon_host = [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] 2024-09-20T23:50:06.444 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-20T23:50:06.445 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:50:06.445 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T23:50:06.462 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[23740]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:06.463 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:06.463 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3036447182' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:06.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[28324]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:06.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:06.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:06 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3036447182' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:06.474 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:50:06.474 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:50:06.542 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:50:06.542 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T23:50:06.573 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:50:06.573 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:50:06.639 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:06 smithi031 ceph-mon[24497]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:06.639 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:50:06.640 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:06 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3036447182' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:06.643 INFO:tasks.cephadm:Adding mgr.y on smithi018 2024-09-20T23:50:06.644 INFO:tasks.cephadm:Adding mgr.x on smithi031 2024-09-20T23:50:06.644 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply mgr '2;smithi018=y;smithi031=x' 2024-09-20T23:50:08.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:08.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: Reconfiguring mon.c (monmap changed)... 2024-09-20T23:50:08.081 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:08.081 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:08.081 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:08.081 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: Reconfiguring daemon mon.c on smithi018 2024-09-20T23:50:08.081 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:07 smithi031 ceph-mon[24497]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:08.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: Reconfiguring mon.c (monmap changed)... 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: Reconfiguring daemon mon.c on smithi018 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[28324]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:08.224 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: Reconfiguring mon.c (monmap changed)... 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: Reconfiguring daemon mon.c on smithi018 2024-09-20T23:50:08.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:07 smithi018 ceph-mon[23740]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:08.527 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled mgr update... 2024-09-20T23:50:09.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: Reconfiguring mon.b (monmap changed)... 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: Reconfiguring daemon mon.b on smithi031 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: Saving service mgr spec with placement smithi018=y;smithi031=x;count:2 2024-09-20T23:50:09.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: Reconfiguring mon.b (monmap changed)... 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: Reconfiguring daemon mon.b on smithi031 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: Saving service mgr spec with placement smithi018=y;smithi031=x;count:2 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.577 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: Reconfiguring mon.b (monmap changed)... 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: Reconfiguring daemon mon.b on smithi031 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi031=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: Saving service mgr spec with placement smithi018=y;smithi031=x;count:2 2024-09-20T23:50:09.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:09.583 DEBUG:teuthology.orchestra.run.smithi031:mgr.x> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service 2024-09-20T23:50:09.585 INFO:tasks.cephadm:Deploying OSDs... 2024-09-20T23:50:09.585 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:50:09.586 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T23:50:09.606 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T23:50:09.606 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_1 2024-09-20T23:50:09.665 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T23:50:09.665 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-20 23:49:47.995418570 +0000 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-20 23:47:36.728967663 +0000 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-20 23:47:36.728967663 +0000 2024-09-20T23:50:09.666 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-20 23:47:36.728967663 +0000 2024-09-20T23:50:09.666 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T23:50:09.737 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-20T23:50:09.737 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-20T23:50:09.737 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000132977 s, 3.9 MB/s 2024-09-20T23:50:09.738 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T23:50:09.798 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_2 2024-09-20T23:50:09.857 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T23:50:09.857 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:09.857 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-20 23:49:47.996418559 +0000 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-20 23:47:37.041964217 +0000 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-20 23:47:37.041964217 +0000 2024-09-20T23:50:09.858 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-20 23:47:37.041964217 +0000 2024-09-20T23:50:09.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T23:50:09.927 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-20T23:50:09.927 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-20T23:50:09.927 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000132309 s, 3.9 MB/s 2024-09-20T23:50:09.929 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T23:50:09.989 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_3 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-20 23:49:47.996418559 +0000 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-20 23:47:37.358960728 +0000 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-20 23:47:37.358960728 +0000 2024-09-20T23:50:10.049 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-20 23:47:37.358960728 +0000 2024-09-20T23:50:10.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T23:50:10.117 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-20T23:50:10.117 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-20T23:50:10.118 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000123376 s, 4.1 MB/s 2024-09-20T23:50:10.119 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T23:50:10.178 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_4 2024-09-20T23:50:10.237 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T23:50:10.237 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.237 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-20T23:50:10.237 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.237 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.238 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-20 23:49:47.997418547 +0000 2024-09-20T23:50:10.238 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-20 23:47:37.679957195 +0000 2024-09-20T23:50:10.238 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-20 23:47:37.679957195 +0000 2024-09-20T23:50:10.238 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-20 23:47:37.679957195 +0000 2024-09-20T23:50:10.238 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T23:50:10.304 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-20T23:50:10.304 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-20T23:50:10.304 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000130823 s, 3.9 MB/s 2024-09-20T23:50:10.306 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T23:50:10.365 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:50:10.366 DEBUG:teuthology.orchestra.run.smithi031:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T23:50:10.383 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T23:50:10.383 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_1 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 5h/5d Inode: 670 Links: 1 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.439 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2024-09-20 23:49:57.005790255 +0000 2024-09-20T23:50:10.440 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2024-09-20 23:47:31.682621132 +0000 2024-09-20T23:50:10.440 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2024-09-20 23:47:31.682621132 +0000 2024-09-20T23:50:10.440 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: 2024-09-20 23:47:31.682621132 +0000 2024-09-20T23:50:10.440 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T23:50:10.503 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2024-09-20T23:50:10.503 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2024-09-20T23:50:10.503 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000204609 s, 2.5 MB/s 2024-09-20T23:50:10.504 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T23:50:10.560 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_2 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2024-09-20 23:49:57.005790255 +0000 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2024-09-20 23:47:32.010615575 +0000 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2024-09-20 23:47:32.010615575 +0000 2024-09-20T23:50:10.619 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: 2024-09-20 23:47:32.010615575 +0000 2024-09-20T23:50:10.619 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T23:50:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:10.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T23:50:10.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:10.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.683 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2024-09-20T23:50:10.684 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2024-09-20T23:50:10.684 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000220772 s, 2.3 MB/s 2024-09-20T23:50:10.685 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T23:50:10.742 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_3 2024-09-20T23:50:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2024-09-20 23:49:57.006790238 +0000 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2024-09-20 23:47:32.344609917 +0000 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2024-09-20 23:47:32.344609917 +0000 2024-09-20T23:50:10.767 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: 2024-09-20 23:47:32.344609917 +0000 2024-09-20T23:50:10.767 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T23:50:10.831 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2024-09-20T23:50:10.832 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2024-09-20T23:50:10.832 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000269878 s, 1.9 MB/s 2024-09-20T23:50:10.833 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T23:50:10.890 DEBUG:teuthology.orchestra.run.smithi031:> stat /dev/vg_nvme/lv_4 2024-09-20T23:50:10.946 INFO:teuthology.orchestra.run.smithi031.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T23:50:10.946 INFO:teuthology.orchestra.run.smithi031.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T23:50:10.946 INFO:teuthology.orchestra.run.smithi031.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-20T23:50:10.946 INFO:teuthology.orchestra.run.smithi031.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T23:50:10.947 INFO:teuthology.orchestra.run.smithi031.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T23:50:10.947 INFO:teuthology.orchestra.run.smithi031.stdout:Access: 2024-09-20 23:49:57.006790238 +0000 2024-09-20T23:50:10.947 INFO:teuthology.orchestra.run.smithi031.stdout:Modify: 2024-09-20 23:47:32.676604292 +0000 2024-09-20T23:50:10.947 INFO:teuthology.orchestra.run.smithi031.stdout:Change: 2024-09-20 23:47:32.676604292 +0000 2024-09-20T23:50:10.947 INFO:teuthology.orchestra.run.smithi031.stdout: Birth: 2024-09-20 23:47:32.676604292 +0000 2024-09-20T23:50:10.947 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T23:50:11.012 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records in 2024-09-20T23:50:11.012 INFO:teuthology.orchestra.run.smithi031.stderr:1+0 records out 2024-09-20T23:50:11.012 INFO:teuthology.orchestra.run.smithi031.stderr:512 bytes copied, 0.000242431 s, 2.1 MB/s 2024-09-20T23:50:11.013 DEBUG:teuthology.orchestra.run.smithi031:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T23:50:11.071 INFO:tasks.cephadm:Deploying osd.0 on smithi018 with /dev/vg_nvme/lv_4... 2024-09-20T23:50:11.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T23:50:11.645 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:11 smithi031 ceph-mon[24497]: Deploying daemon mgr.x on smithi031 2024-09-20T23:50:11.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:11 smithi018 ceph-mon[23740]: Deploying daemon mgr.x on smithi031 2024-09-20T23:50:11.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:11 smithi018 ceph-mon[28324]: Deploying daemon mgr.x on smithi031 2024-09-20T23:50:12.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:12.309+0000 7fdec4b4b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-mon[24497]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:12.508+0000 7fdec4b4b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:12.612+0000 7fdec4b4b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:50:12.805 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:12.747+0000 7fdec4b4b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:50:12.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[23740]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[28324]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:13.076 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:12.971+0000 7fdec4b4b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:50:13.592 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:13.327+0000 7fdec4b4b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:50:13.593 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:13.419+0000 7fdec4b4b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:50:13.593 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:13.591+0000 7fdec4b4b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:50:13.672 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:50:13.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_4 2024-09-20T23:50:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: Reconfiguring mgr.y (unknown last config time)... 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: Reconfiguring daemon mgr.y on smithi018 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: Reconfiguring mgr.y (unknown last config time)... 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:14.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: Reconfiguring daemon mgr.y on smithi018 2024-09-20T23:50:14.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:14 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.700 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:14.700 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: Reconfiguring mgr.y (unknown last config time)... 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: Reconfiguring daemon mgr.y on smithi018 2024-09-20T23:50:14.701 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:15.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:14.699+0000 7fdec4b4b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:50:15.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:14.792+0000 7fdec4b4b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:50:16.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:15.736+0000 7fdec4b4b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:50:16.397 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.065+0000 7fdec4b4b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:50:16.397 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.203+0000 7fdec4b4b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:50:16.397 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.294+0000 7fdec4b4b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:50:16.397 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.395+0000 7fdec4b4b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.485+0000 7fdec4b4b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.638+0000 7fdec4b4b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.733+0000 7fdec4b4b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.735 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:16.736 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:16.736 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:16.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:16.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:16.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:16.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:16.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:16.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:17.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:16.832+0000 7fdec4b4b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:50:17.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:17.126+0000 7fdec4b4b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:50:17.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:17.217+0000 7fdec4b4b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:50:17.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:17.308+0000 7fdec4b4b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:50:17.855 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:17 smithi018 ceph-mon[23740]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:17.855 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:17 smithi018 ceph-mon[28324]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:18.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:17 smithi031 ceph-mon[24497]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:18.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:50:18.416+0000 7fdec4b4b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:50:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:18 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:50:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-20T23:50:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:50:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:50:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:18 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/4094341624' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: mgrmap e14: y(active, since 59s), standbys: x 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2917250746' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]': finished 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: osdmap e5: 1 total, 0 up, 1 in 2024-09-20T23:50:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3783089292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: mgrmap e14: y(active, since 59s), standbys: x 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2917250746' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]': finished 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: osdmap e5: 1 total, 0 up, 1 in 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:50:19.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:50:19.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:19 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3783089292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: mgrmap e14: y(active, since 59s), standbys: x 2024-09-20T23:50:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2917250746' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9defeb48-e700-4bee-9f7c-af513722ea2e"}]': finished 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: osdmap e5: 1 total, 0 up, 1 in 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:50:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:19 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3783089292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:20 smithi018 ceph-mon[23740]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:20 smithi018 ceph-mon[28324]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:20 smithi031 ceph-mon[24497]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:22.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:22 smithi018 ceph-mon[23740]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:22.743 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:22 smithi018 ceph-mon[28324]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:23.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:22 smithi031 ceph-mon[24497]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[23740]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[28324]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T23:50:24.923 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:24 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:24 smithi031 ceph-mon[24497]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:24 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T23:50:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:24 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:25.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:25 smithi018 ceph-mon[23740]: Deploying daemon osd.0 on smithi018 2024-09-20T23:50:25.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:25 smithi018 ceph-mon[28324]: Deploying daemon osd.0 on smithi018 2024-09-20T23:50:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:25 smithi031 ceph-mon[24497]: Deploying daemon osd.0 on smithi018 2024-09-20T23:50:26.760 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:26 smithi018 ceph-mon[23740]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:26.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:26 smithi018 ceph-mon[28324]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:26 smithi031 ceph-mon[24497]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:28 smithi018 ceph-mon[23740]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:28 smithi018 ceph-mon[28324]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:28 smithi031 ceph-mon[24497]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:30.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[23740]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:30.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:30.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[28324]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:30 smithi031 ceph-mon[24497]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:32.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[23740]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:32.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:32.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:32.613 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[28324]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:32.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:32.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:32.895 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 0 on host 'smithi018' 2024-09-20T23:50:33.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:32 smithi031 ceph-mon[24497]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:33.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:33.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:33.467 DEBUG:teuthology.orchestra.run.smithi018:osd.0> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.0.service 2024-09-20T23:50:33.470 INFO:tasks.cephadm:Deploying osd.1 on smithi018 with /dev/vg_nvme/lv_3... 2024-09-20T23:50:33.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[23740]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[28324]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:33.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:33.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:33.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:34.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:33 smithi031 ceph-mon[24497]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T23:50:34.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:34.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:34.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:34.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[28324]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[28324]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[28324]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[28324]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[23740]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:34.699 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[23740]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T23:50:34.700 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[23740]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T23:50:34.700 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:34.700 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-mon[23740]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:34.700 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[31152]: 2024-09-20T23:50:34.548+0000 7ff94a27e700 -1 osd.0 0 waiting for initial osdmap 2024-09-20T23:50:34.700 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:50:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[31152]: 2024-09-20T23:50:34.551+0000 7ff943c13700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:50:35.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:34 smithi031 ceph-mon[24497]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:35.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:34 smithi031 ceph-mon[24497]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T23:50:35.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:34 smithi031 ceph-mon[24497]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T23:50:35.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:34 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:35.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:34 smithi031 ceph-mon[24497]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:35.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[23740]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:50:35.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[23740]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T23:50:35.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:35.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:35.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[28324]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:50:35.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[28324]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T23:50:35.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:35.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:35 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:36.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:35 smithi031 ceph-mon[24497]: from='osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:50:36.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:35 smithi031 ceph-mon[24497]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T23:50:36.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:35 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:36.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:35 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836] boot 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: osdmap e8: 1 total, 1 up, 1 in 2024-09-20T23:50:36.795 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836] boot 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: osdmap e8: 1 total, 1 up, 1 in 2024-09-20T23:50:36.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:50:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:50:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T23:50:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: osd.0 [v2:172.21.15.18:6802/2470674836,v1:172.21.15.18:6803/2470674836] boot 2024-09-20T23:50:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: osdmap e8: 1 total, 1 up, 1 in 2024-09-20T23:50:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[28324]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:37.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[23740]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T23:50:37.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:37.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:50:37.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:37.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:37 smithi031 ceph-mon[24497]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T23:50:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:37 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:37 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:50:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:37 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:37 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:38.124 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:50:38.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_3 2024-09-20T23:50:38.797 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi018 2024-09-20T23:50:38.797 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi018 to 16068M 2024-09-20T23:50:38.797 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[23740]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:38.797 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi018 2024-09-20T23:50:38.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi018 to 16068M 2024-09-20T23:50:38.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:38 smithi018 ceph-mon[28324]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:38 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi018 2024-09-20T23:50:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:38 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi018 to 16068M 2024-09-20T23:50:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:38 smithi031 ceph-mon[24497]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[28324]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[23740]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:40.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:41.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:40 smithi031 ceph-mon[24497]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:41.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:50:41.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:50:41.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:41.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:41 smithi018 ceph-mon[28324]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:41.588 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:41 smithi018 ceph-mon[23740]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:42.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:41 smithi031 ceph-mon[24497]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:50:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2263645346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]': finished 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: osdmap e10: 2 total, 1 up, 2 in 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2263645346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]': finished 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: osdmap e10: 2 total, 1 up, 2 in 2024-09-20T23:50:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:42 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2263645346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]: dispatch 2024-09-20T23:50:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7417173-0246-410f-accd-8123b778cfaa"}]': finished 2024-09-20T23:50:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: osdmap e10: 2 total, 1 up, 2 in 2024-09-20T23:50:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:42 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:43 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/695524531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:43.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:43 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/695524531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:44.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:43 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/695524531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:50:44.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:44 smithi018 ceph-mon[23740]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:44 smithi018 ceph-mon[28324]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:44 smithi031 ceph-mon[24497]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:46.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:46 smithi018 ceph-mon[23740]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:46.849 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:46 smithi018 ceph-mon[28324]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:46 smithi031 ceph-mon[24497]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:48.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:48 smithi018 ceph-mon[23740]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:48.599 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:48 smithi018 ceph-mon[28324]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:48 smithi031 ceph-mon[24497]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:49.878 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T23:50:49.878 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:49.878 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[23740]: Deploying daemon osd.1 on smithi018 2024-09-20T23:50:49.879 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T23:50:49.879 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:49.879 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:49 smithi018 ceph-mon[28324]: Deploying daemon osd.1 on smithi018 2024-09-20T23:50:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T23:50:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:49 smithi031 ceph-mon[24497]: Deploying daemon osd.1 on smithi018 2024-09-20T23:50:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:50 smithi018 ceph-mon[23740]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:50.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:50 smithi018 ceph-mon[28324]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:51.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:50 smithi031 ceph-mon[24497]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:52.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:52 smithi018 ceph-mon[23740]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:52.667 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:52 smithi018 ceph-mon[28324]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:53.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:52 smithi031 ceph-mon[24497]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:54.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:54 smithi031 ceph-mon[24497]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:54.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:54.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:54.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:54.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[23740]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[28324]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[23740]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[28324]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:56.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:56 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:56.978 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 1 on host 'smithi018' 2024-09-20T23:50:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:56 smithi031 ceph-mon[24497]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:56 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:56 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:57.615 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:57.616 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:57.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:57.640 DEBUG:teuthology.orchestra.run.smithi018:osd.1> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.1.service 2024-09-20T23:50:57.642 INFO:tasks.cephadm:Deploying osd.2 on smithi018 with /dev/vg_nvme/lv_2... 2024-09-20T23:50:57.643 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T23:50:58.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:58.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T23:50:58.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:50:58.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:50:58.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:50:58.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[23740]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:58.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T23:50:58.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T23:50:58.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:58.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:58.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-mon[28324]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:58.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[34112]: 2024-09-20T23:50:58.516+0000 7f1518cbd700 -1 osd.1 0 waiting for initial osdmap 2024-09-20T23:50:58.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:50:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[34112]: 2024-09-20T23:50:58.519+0000 7f1512e53700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: from='osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:58.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:58 smithi031 ceph-mon[24497]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[23740]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[23740]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[28324]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[28324]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:50:59.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:50:59 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:59 smithi031 ceph-mon[24497]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:59 smithi031 ceph-mon[24497]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T23:51:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:59 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:50:59 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:51:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:51:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:51:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860] boot 2024-09-20T23:51:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: osdmap e13: 2 total, 2 up, 2 in 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860] boot 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: osdmap e13: 2 total, 2 up, 2 in 2024-09-20T23:51:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:00 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:51:01.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:51:01.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T23:51:01.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: osd.1 [v2:172.21.15.18:6810/4101436860,v1:172.21.15.18:6811/4101436860] boot 2024-09-20T23:51:01.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: osdmap e13: 2 total, 2 up, 2 in 2024-09-20T23:51:01.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:00 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:51:01.340 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:51:01.362 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_2 2024-09-20T23:51:01.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:01 smithi018 ceph-mon[28324]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T23:51:01.635 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:01 smithi018 ceph-mon[23740]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T23:51:02.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:01 smithi031 ceph-mon[24497]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T23:51:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:02 smithi018 ceph-mon[23740]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:02.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:02 smithi018 ceph-mon[28324]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:02 smithi031 ceph-mon[24497]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:04.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi018 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi018 to 8034M 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:04.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:04 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:04.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:04.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi018 2024-09-20T23:51:04.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi018 to 8034M 2024-09-20T23:51:04.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.756 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:04.756 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:04.756 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:04.756 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:04.756 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi018 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi018 to 8034M 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:04.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:04.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:04.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:04 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:05.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:05 smithi031 ceph-mon[24497]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:05.761 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:05 smithi018 ceph-mon[28324]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:05.761 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:05 smithi018 ceph-mon[23740]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:06.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:06.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/203260944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]': finished 2024-09-20T23:51:06.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: osdmap e15: 3 total, 2 up, 3 in 2024-09-20T23:51:06.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/203260944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]': finished 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: osdmap e15: 3 total, 2 up, 3 in 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/203260944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]: dispatch 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de327d19-2a00-4f10-8560-1897f03bd6ed"}]': finished 2024-09-20T23:51:06.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: osdmap e15: 3 total, 2 up, 3 in 2024-09-20T23:51:06.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:07 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/74716646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:07.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:07 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/74716646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:07.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:07 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/74716646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:08.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:08 smithi031 ceph-mon[24497]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:08 smithi018 ceph-mon[23740]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:08 smithi018 ceph-mon[28324]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:10 smithi018 ceph-mon[23740]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:10.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:10 smithi018 ceph-mon[28324]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:10 smithi031 ceph-mon[24497]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:12.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:12 smithi018 ceph-mon[23740]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:12.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:12 smithi018 ceph-mon[28324]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:12.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:12 smithi031 ceph-mon[24497]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T23:51:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T23:51:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T23:51:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:14.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:14 smithi018 ceph-mon[23740]: Deploying daemon osd.2 on smithi018 2024-09-20T23:51:14.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:14 smithi018 ceph-mon[23740]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:14.606 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:14 smithi018 ceph-mon[28324]: Deploying daemon osd.2 on smithi018 2024-09-20T23:51:14.606 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:14 smithi018 ceph-mon[28324]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:14.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:14 smithi031 ceph-mon[24497]: Deploying daemon osd.2 on smithi018 2024-09-20T23:51:14.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:14 smithi031 ceph-mon[24497]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:16 smithi031 ceph-mon[24497]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:16.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:16 smithi018 ceph-mon[23740]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:16.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:16 smithi018 ceph-mon[28324]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:18.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[28324]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:18.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:18.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:18.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[23740]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:18.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:18 smithi031 ceph-mon[24497]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:18.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:19.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:51:19.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:51:19.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:51:19.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:51:19.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:51:19.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:51:20.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:20 smithi018 ceph-mon[23740]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:20.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:20 smithi018 ceph-mon[28324]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:20 smithi031 ceph-mon[24497]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:21.459 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 2 on host 'smithi018' 2024-09-20T23:51:22.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:22.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:21 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:22.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:22.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:21 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:22.105 DEBUG:teuthology.orchestra.run.smithi018:osd.2> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service 2024-09-20T23:51:22.108 INFO:tasks.cephadm:Deploying osd.3 on smithi018 with /dev/vg_nvme/lv_1... 2024-09-20T23:51:22.108 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[23740]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[23740]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[23740]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[23740]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[28324]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T23:51:22.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[28324]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T23:51:22.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:22.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[28324]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:22.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-mon[28324]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:22.969 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[37014]: 2024-09-20T23:51:22.713+0000 7f58447bf700 -1 osd.2 0 waiting for initial osdmap 2024-09-20T23:51:22.969 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:22 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[37014]: 2024-09-20T23:51:22.716+0000 7f583f156700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:51:23.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:22 smithi031 ceph-mon[24497]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T23:51:23.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:22 smithi031 ceph-mon[24497]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T23:51:23.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:22 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:23.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:22 smithi031 ceph-mon[24497]: from='osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:23.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:22 smithi031 ceph-mon[24497]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:23 smithi031 ceph-mon[24497]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:23 smithi031 ceph-mon[24497]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T23:51:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:23 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:23 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:23 smithi031 ceph-mon[24497]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:24.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[23740]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:24.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[23740]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T23:51:24.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[23740]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[28324]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[28324]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:23 smithi018 ceph-mon[28324]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T23:51:24.855 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:51:24.855 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[23740]: osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803] boot 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[23740]: osdmap e18: 3 total, 3 up, 3 in 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[28324]: osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803] boot 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[28324]: osdmap e18: 3 total, 3 up, 3 in 2024-09-20T23:51:24.856 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:24 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:24 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:51:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:24 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:51:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:24 smithi031 ceph-mon[24497]: osd.2 [v2:172.21.15.18:6818/864272803,v1:172.21.15.18:6819/864272803] boot 2024-09-20T23:51:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:24 smithi031 ceph-mon[24497]: osdmap e18: 3 total, 3 up, 3 in 2024-09-20T23:51:25.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:24 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:51:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:25 smithi031 ceph-mon[24497]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T23:51:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:25 smithi031 ceph-mon[24497]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:25 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T23:51:26.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[23740]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T23:51:26.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[23740]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:26.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T23:51:26.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[28324]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T23:51:26.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[28324]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:26.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:25 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T23:51:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T23:51:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi018 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi018 to 5356M 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.133 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T23:51:27.133 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T23:51:27.133 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi018 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi018 to 5356M 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi018 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi018 to 5356M 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:27.424 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38238]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:27.424 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38238]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:27.424 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38238]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:27.431 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:51:27.450 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_1 2024-09-20T23:51:27.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T23:51:27.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[23740]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T23:51:27.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[23740]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:27.995 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38286]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:27.995 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38286]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:27.995 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 20 23:51:27 smithi018 sudo[38286]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:27.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T23:51:27.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[28324]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T23:51:27.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:27 smithi018 ceph-mon[28324]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:27 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T23:51:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:27 smithi031 ceph-mon[24497]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T23:51:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:27 smithi031 ceph-mon[24497]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:28.340 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38305]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:28.340 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38305]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:28.340 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38305]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:28.908 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[28324]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-20T23:51:28.909 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[28324]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T23:51:28.909 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:28.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[23740]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-20T23:51:28.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[23740]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T23:51:28.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:28 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:28.909 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38312]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:28.909 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38312]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:28.910 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 20 23:51:28 smithi018 sudo[38312]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:28 smithi031 ceph-mon[24497]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-20T23:51:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:28 smithi031 ceph-mon[24497]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T23:51:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:28 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:29.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38340]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:29.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38340]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:29.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38340]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:29.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38422]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:29.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38422]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:29.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 20 23:51:29 smithi018 sudo[38422]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:30.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38461]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:30.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38461]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:30.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38461]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38466]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38466]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38466]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38493]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38493]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:30 smithi018 sudo[38493]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:31.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:31 smithi018 sudo[38511]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:31.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:31 smithi018 sudo[38511]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:31.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:31 smithi018 sudo[38511]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:31 smithi031 sudo[25997]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T23:51:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:31 smithi031 sudo[25997]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:31 smithi031 sudo[25997]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:32.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 sudo[26000]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 sudo[26000]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 sudo[26000]: pam_unix(sudo:session): session closed for user root 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:32.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:32.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:51:32.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:51:32.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:51:33.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:33.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:33.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2814138160' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]': finished 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: osdmap e23: 4 total, 3 up, 4 in 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/876594307' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2814138160' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]': finished 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: osdmap e23: 4 total, 3 up, 4 in 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:33.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:33 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/876594307' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2814138160' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]: dispatch 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c90fbbf-13c5-4db7-a236-2e696cfcb975"}]': finished 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: osdmap e23: 4 total, 3 up, 4 in 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:33.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:33 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/876594307' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:34.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:34 smithi018 ceph-mon[23740]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:34.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:34 smithi018 ceph-mon[28324]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:34.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:34 smithi031 ceph-mon[24497]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:36.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:36 smithi018 ceph-mon[28324]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:36.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:36 smithi018 ceph-mon[23740]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:36 smithi031 ceph-mon[24497]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:38.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[28324]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T23:51:38.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T23:51:38.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:38.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[23740]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T23:51:38.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T23:51:38.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:38 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:38 smithi031 ceph-mon[24497]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T23:51:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:38 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T23:51:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:38 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:39.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:39 smithi018 ceph-mon[23740]: Deploying daemon osd.3 on smithi018 2024-09-20T23:51:39.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:39 smithi018 ceph-mon[28324]: Deploying daemon osd.3 on smithi018 2024-09-20T23:51:40.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:39 smithi031 ceph-mon[24497]: Deploying daemon osd.3 on smithi018 2024-09-20T23:51:40.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:40 smithi018 ceph-mon[23740]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T23:51:40.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:40 smithi018 ceph-mon[28324]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T23:51:41.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:40 smithi031 ceph-mon[24497]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T23:51:42.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:42 smithi018 ceph-mon[23740]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T23:51:42.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:42 smithi018 ceph-mon[28324]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T23:51:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:42 smithi031 ceph-mon[24497]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T23:51:44.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[23740]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-20T23:51:44.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[28324]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:44.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:44.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:44 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:44 smithi031 ceph-mon[24497]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-20T23:51:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:44 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:44 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:44 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:44 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:46.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:46 smithi018 ceph-mon[23740]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:46.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:46 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:46.780 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:46 smithi018 ceph-mon[28324]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:46.781 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:46 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:46 smithi031 ceph-mon[24497]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:46 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:47.213 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 3 on host 'smithi018' 2024-09-20T23:51:47.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:47 smithi018 ceph-mon[23740]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T23:51:47.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:47 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:47.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:47 smithi018 ceph-mon[28324]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T23:51:47.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:47 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:47.800 DEBUG:teuthology.orchestra.run.smithi018:osd.3> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service 2024-09-20T23:51:47.802 INFO:tasks.cephadm:Deploying osd.4 on smithi031 with /dev/vg_nvme/lv_4... 2024-09-20T23:51:47.803 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T23:51:47.831 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:47 smithi031 ceph-mon[24497]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T23:51:47.832 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:47 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:48.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:48 smithi031 ceph-mon[24497]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:48.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:48 smithi031 ceph-mon[24497]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T23:51:48.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:48 smithi031 ceph-mon[24497]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T23:51:48.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:48 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:48.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:48 smithi031 ceph-mon[24497]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:48.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[23740]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:48.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[23740]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T23:51:48.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[23740]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T23:51:48.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:48.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[23740]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[28324]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T23:51:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[28324]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T23:51:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[28324]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T23:51:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-mon[28324]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-20T23:51:48.833 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[39980]: 2024-09-20T23:51:48.556+0000 7f794efb9700 -1 osd.3 0 waiting for initial osdmap 2024-09-20T23:51:48.833 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 20 23:51:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[39980]: 2024-09-20T23:51:48.561+0000 7f794b954700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:51:49.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:49 smithi031 ceph-mon[24497]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:49.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:49 smithi031 ceph-mon[24497]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T23:51:49.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:49.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[28324]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[28324]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[23740]: from='osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-20T23:51:49.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[23740]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T23:51:49.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:49.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681] boot 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: osdmap e26: 4 total, 4 up, 4 in 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:50.591 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681] boot 2024-09-20T23:51:50.592 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: osdmap e26: 4 total, 4 up, 4 in 2024-09-20T23:51:50.592 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:50 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:50.663 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:51:50.663 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:51:50.664 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T23:51:50.664 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: osd.3 [v2:172.21.15.18:6826/3469350681,v1:172.21.15.18:6827/3469350681] boot 2024-09-20T23:51:50.664 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: osdmap e26: 4 total, 4 up, 4 in 2024-09-20T23:51:50.664 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:50 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:51:50.669 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:51:50.691 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi031:vg_nvme/lv_4 2024-09-20T23:51:51.861 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi018 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi018 to 4017M 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.862 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:51.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:51.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:51.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:51 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T23:51:51.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi018 2024-09-20T23:51:51.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi018 to 4017M 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:51.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi018 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi018 to 4017M 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:51:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:51:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:51 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:51:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:52 smithi031 ceph-mon[24497]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:52 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:52 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:52 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[28324]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[23740]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:52.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:51:52.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:51:52.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:52 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:51:53.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:53 smithi031 ceph-mon[24497]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:53.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:53 smithi018 ceph-mon[28324]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:53.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:53 smithi018 ceph-mon[23740]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:51:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:54 smithi018 ceph-mon[23740]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:54 smithi018 ceph-mon[28324]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:54 smithi031 ceph-mon[24497]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/2356030519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]': finished 2024-09-20T23:51:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: osdmap e28: 5 total, 4 up, 5 in 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/3836553801' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/2356030519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]': finished 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: osdmap e28: 5 total, 4 up, 5 in 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:51:55.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/3836553801' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:56.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/2356030519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:56.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]: dispatch 2024-09-20T23:51:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ddc7cb16-9005-476e-be8a-7fd31166d6c8"}]': finished 2024-09-20T23:51:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: osdmap e28: 5 total, 4 up, 5 in 2024-09-20T23:51:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:51:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/3836553801' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:51:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:56 smithi018 ceph-mon[28324]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:56 smithi018 ceph-mon[23740]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:56 smithi031 ceph-mon[24497]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:58.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:51:58 smithi031 ceph-mon[24497]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:51:58 smithi018 ceph-mon[23740]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:51:58.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:51:58 smithi018 ceph-mon[28324]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:00.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:00 smithi018 ceph-mon[23740]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:00 smithi018 ceph-mon[28324]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:00 smithi031 ceph-mon[24497]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:01.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T23:52:01.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:01.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[28324]: Deploying daemon osd.4 on smithi031 2024-09-20T23:52:01.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T23:52:01.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:01.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:01 smithi018 ceph-mon[23740]: Deploying daemon osd.4 on smithi031 2024-09-20T23:52:01.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:01 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T23:52:01.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:01 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:01.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:01 smithi031 ceph-mon[24497]: Deploying daemon osd.4 on smithi031 2024-09-20T23:52:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:02 smithi018 ceph-mon[23740]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:02.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:02 smithi018 ceph-mon[28324]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:02 smithi031 ceph-mon[24497]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:04.834 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:04 smithi031 ceph-mon[24497]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:04 smithi018 ceph-mon[28324]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:04 smithi018 ceph-mon[23740]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[23740]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[28324]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:06.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:06 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:06 smithi031 ceph-mon[24497]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:06 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:08.653 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:08 smithi031 ceph-mon[24497]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:08.653 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:08 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.653 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:08 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[23740]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[28324]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:08.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:08 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:08.935 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 4 on host 'smithi031' 2024-09-20T23:52:09.516 DEBUG:teuthology.orchestra.run.smithi031:osd.4> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.4.service 2024-09-20T23:52:09.518 INFO:tasks.cephadm:Deploying osd.5 on smithi031 with /dev/vg_nvme/lv_3... 2024-09-20T23:52:09.518 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:09.784 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:09 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T23:52:09.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:09.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:09.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:09.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:09 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[28324]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:10.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T23:52:10.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T23:52:10.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:10.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:10.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:10 smithi018 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:10.851 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[27828]: 2024-09-20T23:52:10.576+0000 7f04ab4eb700 -1 osd.4 0 waiting for initial osdmap 2024-09-20T23:52:10.851 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[27828]: 2024-09-20T23:52:10.580+0000 7f04a6e84700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:52:10.851 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:10.851 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T23:52:10.852 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T23:52:10.852 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:10.852 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: from='osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:10.852 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:10 smithi031 ceph-mon[24497]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:11.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[23740]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:11.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[23740]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T23:52:11.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:11.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:11.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[28324]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:11.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[28324]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T23:52:11.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:11.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:11 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:11 smithi031 ceph-mon[24497]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:12.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:11 smithi031 ceph-mon[24497]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T23:52:12.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:11 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:11 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.837 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:52:12.837 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:52:12.838 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:12.838 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.838 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590] boot 2024-09-20T23:52:12.838 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: osdmap e31: 5 total, 5 up, 5 in 2024-09-20T23:52:12.838 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:12 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:52:12.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590] boot 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: osdmap e31: 5 total, 5 up, 5 in 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: osd.4 [v2:172.21.15.31:6800/2550433590,v1:172.21.15.31:6801/2550433590] boot 2024-09-20T23:52:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: osdmap e31: 5 total, 5 up, 5 in 2024-09-20T23:52:12.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:12 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:52:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[23740]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T23:52:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:13.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[28324]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:13 smithi031 ceph-mon[24497]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T23:52:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:14.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:14.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:14.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:14.391 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:52:14.411 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi031:vg_nvme/lv_3 2024-09-20T23:52:14.718 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:14 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi031 2024-09-20T23:52:14.718 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:14 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi031 to 17092M 2024-09-20T23:52:14.718 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:14 smithi031 ceph-mon[24497]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:14.718 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:14 smithi031 ceph-mon[24497]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T23:52:14.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi031 2024-09-20T23:52:14.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi031 to 17092M 2024-09-20T23:52:14.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[23740]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:14.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[23740]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T23:52:14.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi031 2024-09-20T23:52:14.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi031 to 17092M 2024-09-20T23:52:14.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[28324]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:14.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:14 smithi018 ceph-mon[28324]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T23:52:15.746 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:15 smithi031 ceph-mon[24497]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T23:52:15.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:15 smithi018 ceph-mon[23740]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T23:52:15.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:15 smithi018 ceph-mon[28324]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T23:52:16.873 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:16 smithi031 ceph-mon[24497]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:16.873 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:16 smithi031 ceph-mon[24497]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T23:52:16.873 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:16.873 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:16.873 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:16 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[23740]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[23740]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[28324]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[28324]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:16 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:17.854 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:17 smithi031 ceph-mon[24497]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:17.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:17 smithi018 ceph-mon[23740]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:17.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:17 smithi018 ceph-mon[28324]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/1966538591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]': finished 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: osdmap e36: 6 total, 5 up, 6 in 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/1966538591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]': finished 2024-09-20T23:52:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: osdmap e36: 6 total, 5 up, 6 in 2024-09-20T23:52:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/1966538591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]: dispatch 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "639f47c5-8bca-438a-b08a-3e5414f86c3f"}]': finished 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: osdmap e36: 6 total, 5 up, 6 in 2024-09-20T23:52:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/2911810425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:52:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:52:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/2911810425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:52:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:52:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:19 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/2911810425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:52:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:52:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:20 smithi018 ceph-mon[23740]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:20 smithi018 ceph-mon[28324]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:20 smithi031 ceph-mon[24497]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:22.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:22 smithi031 ceph-mon[24497]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T23:52:22.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:22 smithi018 ceph-mon[23740]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T23:52:22.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:22 smithi018 ceph-mon[28324]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T23:52:24.740 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:24 smithi031 ceph-mon[24497]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:24.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:24 smithi018 ceph-mon[23740]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:24.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:24 smithi018 ceph-mon[28324]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:25 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T23:52:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:25 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:25.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:25 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T23:52:25.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:25 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:25 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T23:52:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:25 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:26.745 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:26 smithi031 ceph-mon[24497]: Deploying daemon osd.5 on smithi031 2024-09-20T23:52:26.745 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:26 smithi031 ceph-mon[24497]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:26 smithi018 ceph-mon[23740]: Deploying daemon osd.5 on smithi031 2024-09-20T23:52:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:26 smithi018 ceph-mon[23740]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:26 smithi018 ceph-mon[28324]: Deploying daemon osd.5 on smithi031 2024-09-20T23:52:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:26 smithi018 ceph-mon[28324]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:28.884 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:28 smithi031 ceph-mon[24497]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:28 smithi018 ceph-mon[23740]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:28 smithi018 ceph-mon[28324]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:30 smithi031 ceph-mon[24497]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:30.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:30 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:30.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[23740]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:30.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:30.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[28324]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:30 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:32.502 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 5 on host 'smithi031' 2024-09-20T23:52:32.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:32 smithi031 ceph-mon[24497]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:32.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:32 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:32.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:32 smithi018 ceph-mon[23740]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:32.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:32 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:32.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:32 smithi018 ceph-mon[28324]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:32.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:32 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:33.498 DEBUG:teuthology.orchestra.run.smithi031:osd.5> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service 2024-09-20T23:52:33.500 INFO:tasks.cephadm:Deploying osd.6 on smithi031 with /dev/vg_nvme/lv_2... 2024-09-20T23:52:33.501 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T23:52:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:33 smithi031 ceph-mon[24497]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:33 smithi031 ceph-mon[24497]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:33 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[23740]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:33.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[28324]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[28324]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T23:52:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:33 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:34.774 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:34.774 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T23:52:34.774 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T23:52:34.775 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:34.775 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:34.775 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-mon[24497]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:34.775 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[30750]: 2024-09-20T23:52:34.642+0000 7f912b2e0700 -1 osd.5 0 waiting for initial osdmap 2024-09-20T23:52:34.775 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 20 23:52:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[30750]: 2024-09-20T23:52:34.647+0000 7f9126478700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:52:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T23:52:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: from='osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:34 smithi018 ceph-mon[28324]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200] boot 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: osdmap e39: 6 total, 6 up, 6 in 2024-09-20T23:52:35.739 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:35 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200] boot 2024-09-20T23:52:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: osdmap e39: 6 total, 6 up, 6 in 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: osd.5 [v2:172.21.15.31:6808/2129965200,v1:172.21.15.31:6809/2129965200] boot 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: osdmap e39: 6 total, 6 up, 6 in 2024-09-20T23:52:35.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:35 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:52:36.565 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:52:36.587 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi031:vg_nvme/lv_2 2024-09-20T23:52:36.686 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:36 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:52:36.686 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:36 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:52:36.687 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:36 smithi031 ceph-mon[24497]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:36.687 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:36 smithi031 ceph-mon[24497]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T23:52:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:52:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:52:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[23740]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:36.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[23740]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T23:52:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:52:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:52:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[28324]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T23:52:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:36 smithi018 ceph-mon[28324]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T23:52:38.684 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:38 smithi031 ceph-mon[24497]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:38.684 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:38 smithi031 ceph-mon[24497]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T23:52:38.684 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:38 smithi031 ceph-mon[24497]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T23:52:38.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[23740]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:38.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[23740]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T23:52:38.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[23740]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T23:52:38.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[28324]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:38.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[28324]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T23:52:38.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:38 smithi018 ceph-mon[28324]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:39.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:39.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:39 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:40.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:39 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:40.691 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:40.691 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi031 2024-09-20T23:52:40.692 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi031 to 8546M 2024-09-20T23:52:40.692 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T23:52:40.692 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:40.692 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:40.692 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:40 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi031 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi031 to 8546M 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:40.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi031 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi031 to 8546M 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:52:40.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:40 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:41.856 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:41 smithi031 ceph-mon[24497]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:41.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:41 smithi018 ceph-mon[23740]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:41.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:41 smithi018 ceph-mon[28324]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:52:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T23:52:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: Cluster is now healthy 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/1742117138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]': finished 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: osdmap e44: 7 total, 6 up, 7 in 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: Cluster is now healthy 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/1742117138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]': finished 2024-09-20T23:52:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: osdmap e44: 7 total, 6 up, 7 in 2024-09-20T23:52:42.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:42 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: Cluster is now healthy 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/1742117138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]: dispatch 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b318df-6e83-4313-87bc-8d522ab34817"}]': finished 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: osdmap e44: 7 total, 6 up, 7 in 2024-09-20T23:52:43.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:42 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:43 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/131403189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:43.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:43 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/131403189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:44.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:43 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/131403189' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:52:44.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:44 smithi018 ceph-mon[23740]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:44 smithi018 ceph-mon[28324]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:44 smithi031 ceph-mon[24497]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:52:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:46 smithi031 ceph-mon[24497]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:52:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:46 smithi018 ceph-mon[23740]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:52:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:46 smithi018 ceph-mon[28324]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:52:48.716 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:48 smithi031 ceph-mon[24497]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:48 smithi018 ceph-mon[23740]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:48.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:48 smithi018 ceph-mon[28324]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:52:49.656 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T23:52:49.656 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:49 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T23:52:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:49 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T23:52:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:49 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:50.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:50 smithi018 ceph-mon[28324]: Deploying daemon osd.6 on smithi031 2024-09-20T23:52:50.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:50 smithi018 ceph-mon[28324]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T23:52:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:50 smithi018 ceph-mon[23740]: Deploying daemon osd.6 on smithi031 2024-09-20T23:52:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:50 smithi018 ceph-mon[23740]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T23:52:51.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:50 smithi031 ceph-mon[24497]: Deploying daemon osd.6 on smithi031 2024-09-20T23:52:51.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:50 smithi031 ceph-mon[24497]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T23:52:52.827 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:52 smithi031 ceph-mon[24497]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:52 smithi018 ceph-mon[23740]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:52 smithi018 ceph-mon[28324]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[23740]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[28324]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:54.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:54 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:54 smithi031 ceph-mon[24497]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:55.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:55.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:55.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:54 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:56 smithi031 ceph-mon[24497]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:56 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:56 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[23740]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[28324]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:56 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:57.173 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 6 on host 'smithi031' 2024-09-20T23:52:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:57.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:57 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:57.799 DEBUG:teuthology.orchestra.run.smithi031:osd.6> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service 2024-09-20T23:52:57.801 INFO:tasks.cephadm:Deploying osd.7 on smithi031 with /dev/vg_nvme/lv_1... 2024-09-20T23:52:57.801 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T23:52:57.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:57.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:52:57.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:57 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:52:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T23:52:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T23:52:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[23740]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:58.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:58 smithi018 ceph-mon[28324]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:58.975 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[33628]: 2024-09-20T23:52:58.660+0000 7f9003833700 -1 osd.6 0 waiting for initial osdmap 2024-09-20T23:52:58.975 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[33628]: 2024-09-20T23:52:58.667+0000 7f8ffb9c5700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: from='osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:58.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:58 smithi031 ceph-mon[24497]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:52:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[23740]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[23740]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T23:52:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:59.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[28324]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:59.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[28324]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T23:52:59.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:59.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:52:59 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:59.982 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:59 smithi031 ceph-mon[24497]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:52:59.982 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:59 smithi031 ceph-mon[24497]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T23:52:59.982 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:59 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:52:59.982 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:52:59 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:53:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:53:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:53:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526] boot 2024-09-20T23:53:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: osdmap e47: 7 total, 7 up, 7 in 2024-09-20T23:53:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:00 smithi031 ceph-mon[24497]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T23:53:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:53:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:53:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:53:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526] boot 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: osdmap e47: 7 total, 7 up, 7 in 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[28324]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: osd.6 [v2:172.21.15.31:6816/2805350526,v1:172.21.15.31:6817/2805350526] boot 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: osdmap e47: 7 total, 7 up, 7 in 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:00.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:00 smithi018 ceph-mon[23740]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi031 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi031 to 5697M 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[28324]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi031 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi031 to 5697M 2024-09-20T23:53:02.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:02 smithi018 ceph-mon[23740]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T23:53:02.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi031 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi031 to 5697M 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:02 smithi031 ceph-mon[24497]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T23:53:02.768 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-09-20T23:53:02.788 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi031:vg_nvme/lv_1 2024-09-20T23:53:03.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:03 smithi018 ceph-mon[23740]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T23:53:03.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:03 smithi018 ceph-mon[28324]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T23:53:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:03 smithi031 ceph-mon[24497]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T23:53:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[28324]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[28324]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[28324]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T23:53:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[23740]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[23740]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:04 smithi018 ceph-mon[23740]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T23:53:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:04 smithi031 ceph-mon[24497]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:04 smithi031 ceph-mon[24497]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:04 smithi031 ceph-mon[24497]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T23:53:05.951 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:05 smithi031 ceph-mon[24497]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:05.951 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:05 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:53:05.951 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:05 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:53:05.952 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:05 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[28324]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:53:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:53:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[23740]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi031:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:06.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T23:53:06.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T23:53:06.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:05 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:06.679 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:06 smithi031 ceph-mon[24497]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:06 smithi018 ceph-mon[23740]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:06 smithi018 ceph-mon[28324]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/1525939410' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]': finished 2024-09-20T23:53:08.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: osdmap e52: 8 total, 7 up, 8 in 2024-09-20T23:53:08.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:08.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:07 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/115748209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/1525939410' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]': finished 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: osdmap e52: 8 total, 7 up, 8 in 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/115748209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/1525939410' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]: dispatch 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cb97aa0-e5b2-4583-a665-c54670fa6d8f"}]': finished 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: osdmap e52: 8 total, 7 up, 8 in 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:07 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/115748209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T23:53:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:08 smithi031 ceph-mon[24497]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:08 smithi031 ceph-mon[24497]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:08 smithi031 ceph-mon[24497]: Cluster is now healthy 2024-09-20T23:53:09.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[28324]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:09.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[28324]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:09.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[28324]: Cluster is now healthy 2024-09-20T23:53:09.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[23740]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:09.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[23740]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:09.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:08 smithi018 ceph-mon[23740]: Cluster is now healthy 2024-09-20T23:53:11.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:10 smithi031 ceph-mon[24497]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-20T23:53:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:10 smithi018 ceph-mon[23740]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-20T23:53:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:10 smithi018 ceph-mon[28324]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-20T23:53:12.880 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:12 smithi031 ceph-mon[24497]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:12 smithi018 ceph-mon[23740]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:12 smithi018 ceph-mon[28324]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:13.900 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T23:53:13.900 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:13 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T23:53:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:13 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T23:53:14.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:13 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:14.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:14 smithi031 ceph-mon[24497]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-20T23:53:14.861 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:14 smithi031 ceph-mon[24497]: Deploying daemon osd.7 on smithi031 2024-09-20T23:53:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:14 smithi018 ceph-mon[23740]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-20T23:53:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:14 smithi018 ceph-mon[23740]: Deploying daemon osd.7 on smithi031 2024-09-20T23:53:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:14 smithi018 ceph-mon[28324]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-20T23:53:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:14 smithi018 ceph-mon[28324]: Deploying daemon osd.7 on smithi031 2024-09-20T23:53:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:16 smithi031 ceph-mon[24497]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T23:53:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:16 smithi018 ceph-mon[23740]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T23:53:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:16 smithi018 ceph-mon[28324]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T23:53:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:18 smithi031 ceph-mon[24497]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:18 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[23740]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[28324]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:18 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:19.849 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:19.849 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:19 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:19 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:19 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:20.994 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:20 smithi031 ceph-mon[24497]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:20 smithi018 ceph-mon[23740]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:20 smithi018 ceph-mon[28324]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:21.549 INFO:teuthology.orchestra.run.smithi031.stdout:Created osd(s) 7 on host 'smithi031' 2024-09-20T23:53:22.170 DEBUG:teuthology.orchestra.run.smithi031:osd.7> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.7.service 2024-09-20T23:53:22.172 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-20T23:53:22.172 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T23:53:22.662 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:22 smithi031 ceph-mon[24497]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:22.662 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:22 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:22.663 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:22 smithi031 ceph-mon[24497]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.663 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:22 smithi031 ceph-mon[24497]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.663 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:22 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:22.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[28324]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:22.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:22.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[28324]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[28324]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[23740]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[23740]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T23:53:22.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:22 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[28324]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[28324]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[28324]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[28324]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[23740]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:23.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[23740]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:23 smithi018 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-mon[24497]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T23:53:23.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-mon[24497]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T23:53:23.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:23.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-mon[24497]: from='osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.860 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-mon[24497]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-20T23:53:23.861 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[36535]: 2024-09-20T23:53:23.559+0000 7f92a5455700 -1 osd.7 0 waiting for initial osdmap 2024-09-20T23:53:23.861 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 20 23:53:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[36535]: 2024-09-20T23:53:23.567+0000 7f929edea700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T23:53:24.188 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4215568596' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:24 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4215568596' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:24.894 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1726876379,"num_in_osds":8,"osd_in_since":1726876386,"num_remapped_pgs":0} 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]': finished 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:24.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:24 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4215568596' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:25.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: purged_snaps scrub starts 2024-09-20T23:53:25.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: purged_snaps scrub ok 2024-09-20T23:53:25.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008] boot 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: osdmap e55: 8 total, 8 up, 8 in 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[23740]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: purged_snaps scrub starts 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: purged_snaps scrub ok 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008] boot 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: osdmap e55: 8 total, 8 up, 8 in 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:25.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:25 smithi018 ceph-mon[28324]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: purged_snaps scrub starts 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: purged_snaps scrub ok 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: osd.7 [v2:172.21.15.31:6824/121977008,v1:172.21.15.31:6825/121977008] boot 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: osdmap e55: 8 total, 8 up, 8 in 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:25.875 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:25 smithi031 ceph-mon[24497]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T23:53:25.896 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi031 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi031 to 4273M 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[28324]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:26.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi031 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi031 to 4273M 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:26.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:26 smithi018 ceph-mon[23740]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T23:53:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: Detected new or changed devices on smithi031 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi031 to 4273M 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:27.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:27.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:26 smithi031 ceph-mon[24497]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T23:53:27.789 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:27 smithi018 ceph-mon[23740]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:27.790 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:27 smithi018 ceph-mon[23740]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T23:53:27.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:27 smithi018 ceph-mon[28324]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:27.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:27 smithi018 ceph-mon[28324]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T23:53:27.884 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:53:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:27 smithi031 ceph-mon[24497]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T23:53:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:27 smithi031 ceph-mon[24497]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T23:53:28.596 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1726876404,"num_in_osds":8,"osd_in_since":1726876386,"num_remapped_pgs":0} 2024-09-20T23:53:28.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T23:53:28.822 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:53:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[23740]: pgmap v171: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1380372341' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[23740]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T23:53:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[28324]: pgmap v171: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1380372341' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:28 smithi018 ceph-mon[28324]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T23:53:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:28 smithi031 ceph-mon[24497]: pgmap v171: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:28 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1380372341' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T23:53:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:28 smithi031 ceph-mon[24497]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T23:53:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:53:30.611 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":59,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","created":"2024-09-20T23:48:30.812529+0000","modified":"2024-09-20T23:53:28.642503+0000","last_up_change":"2024-09-20T23:53:24.554086+0000","last_in_change":"2024-09-20T23:53:06.829051+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T23:51:25.523618+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9defeb48-e700-4bee-9f7c-af513722ea2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6803","nonce":2470674836}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6805","nonce":2470674836}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6809","nonce":2470674836}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6807","nonce":2470674836}]},"public_addr":"172.21.15.18:6803/2470674836","cluster_addr":"172.21.15.18:6805/2470674836","heartbeat_back_addr":"172.21.15.18:6809/2470674836","heartbeat_front_addr":"172.21.15.18:6807/2470674836","state":["exists","up"]},{"osd":1,"uuid":"a7417173-0246-410f-accd-8123b778cfaa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6811","nonce":4101436860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6813","nonce":4101436860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6817","nonce":4101436860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6815","nonce":4101436860}]},"public_addr":"172.21.15.18:6811/4101436860","cluster_addr":"172.21.15.18:6813/4101436860","heartbeat_back_addr":"172.21.15.18:6817/4101436860","heartbeat_front_addr":"172.21.15.18:6815/4101436860","state":["exists","up"]},{"osd":2,"uuid":"de327d19-2a00-4f10-8560-1897f03bd6ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6819","nonce":864272803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6821","nonce":864272803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6825","nonce":864272803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6823","nonce":864272803}]},"public_addr":"172.21.15.18:6819/864272803","cluster_addr":"172.21.15.18:6821/864272803","heartbeat_back_addr":"172.21.15.18:6825/864272803","heartbeat_front_addr":"172.21.15.18:6823/864272803","state":["exists","up"]},{"osd":3,"uuid":"6c90fbbf-13c5-4db7-a236-2e696cfcb975","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6827","nonce":3469350681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6829","nonce":3469350681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6833","nonce":3469350681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6831","nonce":3469350681}]},"public_addr":"172.21.15.18:6827/3469350681","cluster_addr":"172.21.15.18:6829/3469350681","heartbeat_back_addr":"172.21.15.18:6833/3469350681","heartbeat_front_addr":"172.21.15.18:6831/3469350681","state":["exists","up"]},{"osd":4,"uuid":"ddc7cb16-9005-476e-be8a-7fd31166d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6800","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6801","nonce":2550433590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6803","nonce":2550433590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6807","nonce":2550433590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6805","nonce":2550433590}]},"public_addr":"172.21.15.31:6801/2550433590","cluster_addr":"172.21.15.31:6803/2550433590","heartbeat_back_addr":"172.21.15.31:6807/2550433590","heartbeat_front_addr":"172.21.15.31:6805/2550433590","state":["exists","up"]},{"osd":5,"uuid":"639f47c5-8bca-438a-b08a-3e5414f86c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6809","nonce":2129965200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6811","nonce":2129965200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6815","nonce":2129965200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6813","nonce":2129965200}]},"public_addr":"172.21.15.31:6809/2129965200","cluster_addr":"172.21.15.31:6811/2129965200","heartbeat_back_addr":"172.21.15.31:6815/2129965200","heartbeat_front_addr":"172.21.15.31:6813/2129965200","state":["exists","up"]},{"osd":6,"uuid":"c0b318df-6e83-4313-87bc-8d522ab34817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6817","nonce":2805350526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6818","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6819","nonce":2805350526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6822","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6823","nonce":2805350526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6820","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6821","nonce":2805350526}]},"public_addr":"172.21.15.31:6817/2805350526","cluster_addr":"172.21.15.31:6819/2805350526","heartbeat_back_addr":"172.21.15.31:6823/2805350526","heartbeat_front_addr":"172.21.15.31:6821/2805350526","state":["exists","up"]},{"osd":7,"uuid":"0cb97aa0-e5b2-4583-a665-c54670fa6d8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6824","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6825","nonce":121977008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6826","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6827","nonce":121977008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6830","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6831","nonce":121977008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6828","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6829","nonce":121977008}]},"public_addr":"172.21.15.31:6825/121977008","cluster_addr":"172.21.15.31:6827/121977008","heartbeat_back_addr":"172.21.15.31:6831/121977008","heartbeat_front_addr":"172.21.15.31:6829/121977008","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:33.646996+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:57.612531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:21.810033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:47.753462+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:09.754493+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:33.863531+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:57.824927+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:53:22.657616+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/1509280396":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/451441213":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/155433796":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6800/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/2751113824":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/2767829388":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6800/2006779287":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/3672252612":"2024-09-21T23:49:18.975789+0000","172.21.15.18:6801/2006779287":"2024-09-21T23:48:56.671898+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T23:53:30.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:30 smithi018 ceph-mon[23740]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:30.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:30 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/573713430' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:53:30.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:30 smithi018 ceph-mon[28324]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:30.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:30 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/573713430' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:53:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:30 smithi031 ceph-mon[24497]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T23:53:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:30 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/573713430' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:53:31.370 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-20T23:51:25.523618+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-20T23:53:31.371 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-20T23:53:31.591 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:53:32.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:31 smithi031 ceph-mon[24497]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:32.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:31 smithi031 ceph-mon[24497]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:32.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:31 smithi031 ceph-mon[24497]: Cluster is now healthy 2024-09-20T23:53:32.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[23740]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:32.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[23740]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:32.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[23740]: Cluster is now healthy 2024-09-20T23:53:32.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[28324]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T23:53:32.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[28324]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T23:53:32.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:31 smithi018 ceph-mon[28324]: Cluster is now healthy 2024-09-20T23:53:33.448 INFO:teuthology.orchestra.run.smithi018.stdout:pg_num: 1 2024-09-20T23:53:34.101 INFO:tasks.cephadm:Adding prometheus.a on smithi031 2024-09-20T23:53:34.101 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply prometheus '1;smithi031=a' 2024-09-20T23:53:34.423 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:34 smithi031 ceph-mon[24497]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:53:34.423 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:34 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2005226365' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T23:53:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:34 smithi018 ceph-mon[28324]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:53:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:34 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2005226365' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T23:53:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:34 smithi018 ceph-mon[23740]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T23:53:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:34 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2005226365' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T23:53:36.262 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled prometheus update... 2024-09-20T23:53:36.827 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:36.827 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.827 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:36.828 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:36.828 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:36.828 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.828 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:36 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T23:53:36.852 DEBUG:teuthology.orchestra.run.smithi031:prometheus.a> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service 2024-09-20T23:53:36.854 INFO:tasks.cephadm:Adding node-exporter.a on smithi018 2024-09-20T23:53:36.854 INFO:tasks.cephadm:Adding node-exporter.b on smithi031 2024-09-20T23:53:36.854 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply node-exporter '2;smithi018=a;smithi031=b' 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T23:53:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' 2024-09-20T23:53:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:36 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T23:53:37.473 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ignoring --setuser ceph since I am not root 2024-09-20T23:53:37.473 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ignoring --setgroup ceph since I am not root 2024-09-20T23:53:37.473 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-mgr[25666]: -- 172.21.15.31:0/2758185447 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560c11744340 con 0x560c1171cc00 2024-09-20T23:53:37.473 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:37.470+0000 7fc74643f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-20T23:53:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[23740]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[23740]: Saving service prometheus spec with placement smithi031=a;count:1 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[23740]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[23740]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[28324]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[28324]: Saving service prometheus spec with placement smithi031=a;count:1 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[28324]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mon[28324]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-20T23:53:37.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setuser ceph since I am not root 2024-09-20T23:53:37.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setgroup ceph since I am not root 2024-09-20T23:53:37.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-mgr[23971]: -- 172.21.15.18:0/96379284 <== mon.1 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c48a2f2340 con 0x55c48a2ca800 2024-09-20T23:53:37.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:37.480+0000 7fa96fe84000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-20T23:53:37.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:37.562+0000 7fa96fe84000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:53:37.747 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:37.553+0000 7fc74643f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:53:37.748 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-mon[24497]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:37.748 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-mon[24497]: Saving service prometheus spec with placement smithi031=a;count:1 2024-09-20T23:53:37.748 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-mon[24497]: from='mgr.14152 172.21.15.18:0/3471319503' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T23:53:37.748 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-mon[24497]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-20T23:53:38.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:37.745+0000 7fc74643f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:53:38.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:37.838+0000 7fc74643f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:53:38.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:37.975+0000 7fc74643f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:53:38.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:37.753+0000 7fa96fe84000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:53:38.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:37.843+0000 7fa96fe84000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:53:38.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:37.978+0000 7fa96fe84000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:53:38.412 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:38.198+0000 7fc74643f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:53:38.535 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:38.193+0000 7fa96fe84000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:53:38.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:38.555+0000 7fc74643f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:53:38.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:38.649+0000 7fc74643f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:53:38.797 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:38.533+0000 7fa96fe84000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:53:38.797 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:38.624+0000 7fa96fe84000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:53:39.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:38.795+0000 7fa96fe84000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:53:39.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:38.822+0000 7fc74643f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:53:40.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:39.886+0000 7fa96fe84000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:53:40.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:39.980+0000 7fa96fe84000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:53:40.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:39.917+0000 7fc74643f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:53:40.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:40.012+0000 7fc74643f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:53:41.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:40.956+0000 7fc74643f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:53:41.263 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:40.939+0000 7fa96fe84000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:53:41.263 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.261+0000 7fa96fe84000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:53:41.598 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.402+0000 7fa96fe84000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:53:41.598 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.494+0000 7fa96fe84000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:53:41.598 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.596+0000 7fa96fe84000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:53:41.613 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.277+0000 7fc74643f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:53:41.613 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.416+0000 7fc74643f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:53:41.614 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.508+0000 7fc74643f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:53:41.614 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.611+0000 7fc74643f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:53:41.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.687+0000 7fa96fe84000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:53:41.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.842+0000 7fa96fe84000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:53:41.957 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.703+0000 7fc74643f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:53:41.957 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.857+0000 7fc74643f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:53:42.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:41.954+0000 7fc74643f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:53:42.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:42.055+0000 7fc74643f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:53:42.337 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:41.939+0000 7fa96fe84000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:53:42.337 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:42.039+0000 7fa96fe84000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:53:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:42.335+0000 7fa96fe84000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:53:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:42.427+0000 7fa96fe84000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:53:42.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:42.520+0000 7fa96fe84000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:53:42.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:42.351+0000 7fc74643f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:53:42.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:42.443+0000 7fc74643f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:53:42.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:42.536+0000 7fc74643f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:53:43.893 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:43.637+0000 7fa96fe84000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:53:43.893 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:43] ENGINE Bus STARTING 2024-09-20T23:53:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: Active manager daemon y restarted 2024-09-20T23:53:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: Activating manager daemon y 2024-09-20T23:53:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: Active manager daemon y restarted 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: Activating manager daemon y 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: Standby manager daemon x restarted 2024-09-20T23:53:43.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-20T23:53:43.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:53:43.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:53:43.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:53:43.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:53:44.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: Active manager daemon y restarted 2024-09-20T23:53:44.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: Activating manager daemon y 2024-09-20T23:53:44.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: Standby manager daemon x restarted 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2190448173' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:43.651+0000 7fc74643f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: [20/Sep/2024:23:53:43] ENGINE Bus STARTING 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: CherryPy Checker: 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: The Application mounted at '' has an empty config. 2024-09-20T23:53:44.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: 2024-09-20T23:53:44.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: [20/Sep/2024:23:53:43] ENGINE Serving on http://:::9283 2024-09-20T23:53:44.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:53:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: [20/Sep/2024:23:53:43] ENGINE Bus STARTED 2024-09-20T23:53:44.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: CherryPy Checker: 2024-09-20T23:53:44.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: The Application mounted at '' has an empty config. 2024-09-20T23:53:44.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-20T23:53:44.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:43] ENGINE Serving on http://:::9283 2024-09-20T23:53:44.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:43] ENGINE Bus STARTED 2024-09-20T23:53:44.720 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled node-exporter update... 2024-09-20T23:53:44.771 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:44] ENGINE Bus STARTING 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: mgrmap e17: y(active, starting, since 0.0565595s), standbys: x 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:53:44.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: Manager daemon y is now available 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:44.773 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: mgrmap e17: y(active, starting, since 0.0565595s), standbys: x 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:53:44.774 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:44.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:44.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:44.776 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: mgrmap e17: y(active, starting, since 0.0565595s), standbys: x 2024-09-20T23:53:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T23:53:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: Manager daemon y is now available 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:53:45.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:44] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:53:45.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:53:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [20/Sep/2024:23:53:44] ENGINE Bus STARTED 2024-09-20T23:53:45.675 DEBUG:teuthology.orchestra.run.smithi018:node-exporter.a> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service 2024-09-20T23:53:45.677 DEBUG:teuthology.orchestra.run.smithi031:node-exporter.b> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service 2024-09-20T23:53:45.679 INFO:tasks.cephadm:Adding alertmanager.a on smithi018 2024-09-20T23:53:45.680 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply alertmanager '1;smithi018=a' 2024-09-20T23:53:45.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: [20/Sep/2024:23:53:44] ENGINE Bus STARTING 2024-09-20T23:53:45.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: mgrmap e18: y(active, since 1.06582s), standbys: x 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi031=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: Saving service node-exporter spec with placement smithi018=a;smithi031=b;count:2 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: [20/Sep/2024:23:53:44] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: [20/Sep/2024:23:53:44] ENGINE Bus STARTED 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: [20/Sep/2024:23:53:44] ENGINE Bus STARTING 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: mgrmap e18: y(active, since 1.06582s), standbys: x 2024-09-20T23:53:45.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi031=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: Saving service node-exporter spec with placement smithi018=a;smithi031=b;count:2 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: [20/Sep/2024:23:53:44] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: [20/Sep/2024:23:53:44] ENGINE Bus STARTED 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:45 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.920 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: [20/Sep/2024:23:53:44] ENGINE Bus STARTING 2024-09-20T23:53:45.920 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: mgrmap e18: y(active, since 1.06582s), standbys: x 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi031=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: Saving service node-exporter spec with placement smithi018=a;smithi031=b;count:2 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: [20/Sep/2024:23:53:44] ENGINE Serving on https://172.21.15.18:7150 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: [20/Sep/2024:23:53:44] ENGINE Bus STARTED 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:45.921 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:45 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-20T23:53:46.935 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:46 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-20T23:53:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:46 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:47.904 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled alertmanager update... 2024-09-20T23:53:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:53:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: Deploying daemon prometheus.a on smithi031 2024-09-20T23:53:48.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:53:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: Deploying daemon prometheus.a on smithi031 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: Deploying daemon prometheus.a on smithi031 2024-09-20T23:53:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:48.649 DEBUG:teuthology.orchestra.run.smithi018:alertmanager.a> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service 2024-09-20T23:53:48.652 INFO:tasks.cephadm:Adding grafana.a on smithi031 2024-09-20T23:53:48.652 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph orch apply grafana '1;smithi031=a' 2024-09-20T23:53:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:49 smithi031 ceph-mon[24497]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:49 smithi031 ceph-mon[24497]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:49 smithi031 ceph-mon[24497]: Saving service alertmanager spec with placement smithi018=a;count:1 2024-09-20T23:53:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[23740]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[23740]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[23740]: Saving service alertmanager spec with placement smithi018=a;count:1 2024-09-20T23:53:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[28324]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:49.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[28324]: from='client.24386 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:49.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:49 smithi018 ceph-mon[28324]: Saving service alertmanager spec with placement smithi018=a;count:1 2024-09-20T23:53:51.351 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:51 smithi031 ceph-mon[24497]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:51.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:51 smithi018 ceph-mon[23740]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:51.380 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:51 smithi018 ceph-mon[28324]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:51.550 INFO:teuthology.orchestra.run.smithi031.stdout:Scheduled grafana update... 2024-09-20T23:53:52.846 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:52 smithi031 ceph-mon[24497]: from='client.24392 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:52.846 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:52 smithi031 ceph-mon[24497]: Saving service grafana spec with placement smithi031=a;count:1 2024-09-20T23:53:52.846 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:52 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:52.846 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:52 smithi031 ceph-mon[24497]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[23740]: from='client.24392 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[23740]: Saving service grafana spec with placement smithi031=a;count:1 2024-09-20T23:53:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[23740]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[28324]: from='client.24392 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi031=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:53:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[28324]: Saving service grafana spec with placement smithi031=a;count:1 2024-09-20T23:53:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:53:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:52 smithi018 ceph-mon[28324]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:53.506 DEBUG:teuthology.orchestra.run.smithi031:grafana.a> sudo journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service 2024-09-20T23:53:53.509 INFO:tasks.cephadm:Setting up client nodes... 2024-09-20T23:53:53.509 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-20T23:53:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:54 smithi031 ceph-mon[24497]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:55.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:54 smithi018 ceph-mon[28324]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:55.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:54 smithi018 ceph-mon[23740]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:55.601 INFO:teuthology.orchestra.run.smithi018.stdout:[client.0] 2024-09-20T23:53:55.601 INFO:teuthology.orchestra.run.smithi018.stdout: key = AQATC+5mWRaTIxAA5CbbxjRxW4k9VzCP301w7w== 2024-09-20T23:53:56.260 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-20T23:53:56.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-20T23:53:56.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-20T23:53:56.305 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-20T23:53:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2055156844' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[23740]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2055156844' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:53:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:56 smithi018 ceph-mon[28324]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:56 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2055156844' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:53:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:56 smithi031 ceph-mon[24497]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:58.934 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:58 smithi031 ceph-mon[24497]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:58 smithi018 ceph-mon[23740]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:58 smithi018 ceph-mon[28324]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:53:59.441 INFO:teuthology.orchestra.run.smithi031.stdout:[client.1] 2024-09-20T23:53:59.441 INFO:teuthology.orchestra.run.smithi031.stdout: key = AQAXC+5mcR7xGRAAZDR1QhrQlllGLURy7WxmgQ== 2024-09-20T23:53:59.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:59 smithi031 ceph-mon[24497]: from='client.? 172.21.15.31:0/3469490016' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:59.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:59 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:53:59.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:59 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:53:59.863 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:53:59 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:00.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[28324]: from='client.? 172.21.15.31:0/3469490016' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:54:00.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:54:00.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:54:00.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:00.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[23740]: from='client.? 172.21.15.31:0/3469490016' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:54:00.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T23:54:00.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T23:54:00.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:53:59 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:00.329 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-09-20T23:54:00.330 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-20T23:54:00.330 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-20T23:54:00.369 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-20T23:54:00.369 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-20T23:54:00.369 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph mgr dump --format=json 2024-09-20T23:54:00.613 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:00.675 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:00 smithi018 bash[43023]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-20T23:54:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:00 smithi031 ceph-mon[24497]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:00 smithi031 ceph-mon[24497]: Deploying daemon node-exporter.a on smithi018 2024-09-20T23:54:01.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:00 smithi018 ceph-mon[23740]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:01.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:00 smithi018 ceph-mon[23740]: Deploying daemon node-exporter.a on smithi018 2024-09-20T23:54:01.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:00 smithi018 ceph-mon[28324]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:01.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:00 smithi018 ceph-mon[28324]: Deploying daemon node-exporter.a on smithi018 2024-09-20T23:54:01.026 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:00 smithi018 bash[43023]: Getting image source signatures 2024-09-20T23:54:01.026 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:00 smithi018 bash[43023]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-20T23:54:01.026 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:00 smithi018 bash[43023]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-20T23:54:01.026 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:00 smithi018 bash[43023]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-20T23:54:02.952 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:02 smithi018 ceph-mon[23740]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:02.952 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:02 smithi018 ceph-mon[28324]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:02.952 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:02 smithi018 bash[43023]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-20T23:54:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:02 smithi031 ceph-mon[24497]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:03.272 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 bash[43023]: Writing manifest to image destination 2024-09-20T23:54:03.449 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:03.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3880804690' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:54:03.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:03.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3880804690' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:54:03.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:03.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 podman[43023]: 2024-09-20 23:54:03.524230716 +0000 UTC m=+3.024636593 container create b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:03.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 podman[43023]: 2024-09-20 23:54:03.42541697 +0000 UTC m=+2.925822864 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-20T23:54:04.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:03 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3880804690' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:54:04.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:03 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 podman[43023]: 2024-09-20 23:54:03.944615973 +0000 UTC m=+3.445021860 container init b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 podman[43023]: 2024-09-20 23:54:03.948026537 +0000 UTC m=+3.448432425 container start b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.952Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.952Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.952Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.952Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=arp 2024-09-20T23:54:04.216 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-20T23:54:04.217 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=edac 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-20T23:54:04.218 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=os 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=stat 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-20T23:54:04.219 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=time 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=uname 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.953Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[43219]: ts=2024-09-20T23:54:03.954Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:04 smithi018 bash[43023]: b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 2024-09-20T23:54:04.220 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 20 23:54:04 smithi018 systemd[1]: Started Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:54:04.648 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":19,"active_gid":24353,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6800","nonce":269195622},{"type":"v1","addr":"172.21.15.18:6801","nonce":269195622}]},"active_addr":"172.21.15.18:6801/269195622","active_change":"2024-09-20T23:53:43.640531+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24347,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":1401144154}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2592259683}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2820254179}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":3798661254}]}]}} 2024-09-20T23:54:04.651 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-20T23:54:04.651 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-20T23:54:04.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T23:54:04.867 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:05.104 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:05 smithi031 bash[39851]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-20T23:54:05.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[28324]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:05.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:05.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[28324]: Deploying daemon node-exporter.b on smithi031 2024-09-20T23:54:05.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[23740]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:05.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:05.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:05 smithi018 ceph-mon[23740]: Deploying daemon node-exporter.b on smithi031 2024-09-20T23:54:05.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:05 smithi031 ceph-mon[24497]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:05.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:05.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:05 smithi031 ceph-mon[24497]: Deploying daemon node-exporter.b on smithi031 2024-09-20T23:54:05.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:05 smithi031 bash[39851]: Getting image source signatures 2024-09-20T23:54:05.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:05 smithi031 bash[39851]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-20T23:54:05.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:05 smithi031 bash[39851]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-20T23:54:05.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:05 smithi031 bash[39851]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-20T23:54:06.705 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:06.705 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":60,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","created":"2024-09-20T23:48:30.812529+0000","modified":"2024-09-20T23:53:43.640210+0000","last_up_change":"2024-09-20T23:53:24.554086+0000","last_in_change":"2024-09-20T23:53:06.829051+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T23:51:25.523618+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9defeb48-e700-4bee-9f7c-af513722ea2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6803","nonce":2470674836}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6805","nonce":2470674836}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6809","nonce":2470674836}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6807","nonce":2470674836}]},"public_addr":"172.21.15.18:6803/2470674836","cluster_addr":"172.21.15.18:6805/2470674836","heartbeat_back_addr":"172.21.15.18:6809/2470674836","heartbeat_front_addr":"172.21.15.18:6807/2470674836","state":["exists","up"]},{"osd":1,"uuid":"a7417173-0246-410f-accd-8123b778cfaa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6811","nonce":4101436860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6813","nonce":4101436860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6817","nonce":4101436860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6815","nonce":4101436860}]},"public_addr":"172.21.15.18:6811/4101436860","cluster_addr":"172.21.15.18:6813/4101436860","heartbeat_back_addr":"172.21.15.18:6817/4101436860","heartbeat_front_addr":"172.21.15.18:6815/4101436860","state":["exists","up"]},{"osd":2,"uuid":"de327d19-2a00-4f10-8560-1897f03bd6ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6819","nonce":864272803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6821","nonce":864272803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6825","nonce":864272803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6823","nonce":864272803}]},"public_addr":"172.21.15.18:6819/864272803","cluster_addr":"172.21.15.18:6821/864272803","heartbeat_back_addr":"172.21.15.18:6825/864272803","heartbeat_front_addr":"172.21.15.18:6823/864272803","state":["exists","up"]},{"osd":3,"uuid":"6c90fbbf-13c5-4db7-a236-2e696cfcb975","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6827","nonce":3469350681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6829","nonce":3469350681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6833","nonce":3469350681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6831","nonce":3469350681}]},"public_addr":"172.21.15.18:6827/3469350681","cluster_addr":"172.21.15.18:6829/3469350681","heartbeat_back_addr":"172.21.15.18:6833/3469350681","heartbeat_front_addr":"172.21.15.18:6831/3469350681","state":["exists","up"]},{"osd":4,"uuid":"ddc7cb16-9005-476e-be8a-7fd31166d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6800","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6801","nonce":2550433590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6803","nonce":2550433590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6807","nonce":2550433590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6805","nonce":2550433590}]},"public_addr":"172.21.15.31:6801/2550433590","cluster_addr":"172.21.15.31:6803/2550433590","heartbeat_back_addr":"172.21.15.31:6807/2550433590","heartbeat_front_addr":"172.21.15.31:6805/2550433590","state":["exists","up"]},{"osd":5,"uuid":"639f47c5-8bca-438a-b08a-3e5414f86c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6809","nonce":2129965200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6811","nonce":2129965200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6815","nonce":2129965200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6813","nonce":2129965200}]},"public_addr":"172.21.15.31:6809/2129965200","cluster_addr":"172.21.15.31:6811/2129965200","heartbeat_back_addr":"172.21.15.31:6815/2129965200","heartbeat_front_addr":"172.21.15.31:6813/2129965200","state":["exists","up"]},{"osd":6,"uuid":"c0b318df-6e83-4313-87bc-8d522ab34817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6817","nonce":2805350526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6818","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6819","nonce":2805350526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6822","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6823","nonce":2805350526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6820","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6821","nonce":2805350526}]},"public_addr":"172.21.15.31:6817/2805350526","cluster_addr":"172.21.15.31:6819/2805350526","heartbeat_back_addr":"172.21.15.31:6823/2805350526","heartbeat_front_addr":"172.21.15.31:6821/2805350526","state":["exists","up"]},{"osd":7,"uuid":"0cb97aa0-e5b2-4583-a665-c54670fa6d8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6824","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6825","nonce":121977008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6826","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6827","nonce":121977008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6830","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6831","nonce":121977008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6828","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6829","nonce":121977008}]},"public_addr":"172.21.15.31:6825/121977008","cluster_addr":"172.21.15.31:6827/121977008","heartbeat_back_addr":"172.21.15.31:6831/121977008","heartbeat_front_addr":"172.21.15.31:6829/121977008","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:33.646996+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:57.612531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:21.810033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:47.753462+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:09.754493+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:33.863531+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:57.824927+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:53:22.657616+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/706006390":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6801/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/1509280396":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/2514861664":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6800/586451024":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/451441213":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/155433796":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6801/586451024":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6800/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/2751113824":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/2767829388":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6800/2006779287":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/2320652463":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/513740309":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/3672252612":"2024-09-21T23:49:18.975789+0000","172.21.15.18:6801/2006779287":"2024-09-21T23:48:56.671898+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T23:54:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:06 smithi018 ceph-mon[23740]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:06 smithi018 ceph-mon[28324]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:06.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:06 smithi031 ceph-mon[24497]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:06.961 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:06 smithi031 bash[39851]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-20T23:54:06.961 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:06 smithi031 bash[39851]: Writing manifest to image destination 2024-09-20T23:54:07.411 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-20T23:54:07.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T23:54:07.524 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 podman[39851]: 2024-09-20 23:54:07.224202478 +0000 UTC m=+2.278041405 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-20T23:54:07.524 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 podman[39851]: 2024-09-20 23:54:07.339219504 +0000 UTC m=+2.393058419 container create 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-20T23:54:07.630 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:07 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1740325433' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:07.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:07 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1740325433' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1740325433' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:08.001 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 podman[39851]: 2024-09-20 23:54:07.524242824 +0000 UTC m=+2.578081750 container init 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 podman[39851]: 2024-09-20 23:54:07.52739076 +0000 UTC m=+2.581229675 container start 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.531Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.531Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.532Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.532Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=arp 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-20T23:54:08.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=edac 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-20T23:54:08.003 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=os 2024-09-20T23:54:08.004 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=stat 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=time 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=uname 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-20T23:54:08.005 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-20T23:54:08.006 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-20T23:54:08.006 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[39928]: ts=2024-09-20T23:54:07.533Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-20T23:54:08.006 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 bash[39851]: 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 2024-09-20T23:54:08.006 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 20 23:54:07 smithi031 systemd[1]: Started Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[23740]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:08.870 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:08.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:54:08.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:54:08.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:54:08.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:08 smithi018 ceph-mon[28324]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:08 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:08 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T23:54:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:08 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:54:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:08 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:54:09.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:08 smithi031 ceph-mon[24497]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:09.126 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:09] "GET /metrics HTTP/1.1" 200 191508 "" "Prometheus/2.33.4" 2024-09-20T23:54:09.527 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:09.527 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":60,"fsid":"c1f9e122-77aa-11ef-baf5-efdc52797490","created":"2024-09-20T23:48:30.812529+0000","modified":"2024-09-20T23:53:43.640210+0000","last_up_change":"2024-09-20T23:53:24.554086+0000","last_in_change":"2024-09-20T23:53:06.829051+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T23:51:25.523618+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9defeb48-e700-4bee-9f7c-af513722ea2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6803","nonce":2470674836}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6805","nonce":2470674836}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6809","nonce":2470674836}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2470674836},{"type":"v1","addr":"172.21.15.18:6807","nonce":2470674836}]},"public_addr":"172.21.15.18:6803/2470674836","cluster_addr":"172.21.15.18:6805/2470674836","heartbeat_back_addr":"172.21.15.18:6809/2470674836","heartbeat_front_addr":"172.21.15.18:6807/2470674836","state":["exists","up"]},{"osd":1,"uuid":"a7417173-0246-410f-accd-8123b778cfaa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6811","nonce":4101436860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6813","nonce":4101436860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6817","nonce":4101436860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":4101436860},{"type":"v1","addr":"172.21.15.18:6815","nonce":4101436860}]},"public_addr":"172.21.15.18:6811/4101436860","cluster_addr":"172.21.15.18:6813/4101436860","heartbeat_back_addr":"172.21.15.18:6817/4101436860","heartbeat_front_addr":"172.21.15.18:6815/4101436860","state":["exists","up"]},{"osd":2,"uuid":"de327d19-2a00-4f10-8560-1897f03bd6ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6819","nonce":864272803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6821","nonce":864272803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6825","nonce":864272803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":864272803},{"type":"v1","addr":"172.21.15.18:6823","nonce":864272803}]},"public_addr":"172.21.15.18:6819/864272803","cluster_addr":"172.21.15.18:6821/864272803","heartbeat_back_addr":"172.21.15.18:6825/864272803","heartbeat_front_addr":"172.21.15.18:6823/864272803","state":["exists","up"]},{"osd":3,"uuid":"6c90fbbf-13c5-4db7-a236-2e696cfcb975","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6827","nonce":3469350681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6829","nonce":3469350681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6833","nonce":3469350681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":3469350681},{"type":"v1","addr":"172.21.15.18:6831","nonce":3469350681}]},"public_addr":"172.21.15.18:6827/3469350681","cluster_addr":"172.21.15.18:6829/3469350681","heartbeat_back_addr":"172.21.15.18:6833/3469350681","heartbeat_front_addr":"172.21.15.18:6831/3469350681","state":["exists","up"]},{"osd":4,"uuid":"ddc7cb16-9005-476e-be8a-7fd31166d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6800","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6801","nonce":2550433590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6802","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6803","nonce":2550433590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6806","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6807","nonce":2550433590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6804","nonce":2550433590},{"type":"v1","addr":"172.21.15.31:6805","nonce":2550433590}]},"public_addr":"172.21.15.31:6801/2550433590","cluster_addr":"172.21.15.31:6803/2550433590","heartbeat_back_addr":"172.21.15.31:6807/2550433590","heartbeat_front_addr":"172.21.15.31:6805/2550433590","state":["exists","up"]},{"osd":5,"uuid":"639f47c5-8bca-438a-b08a-3e5414f86c3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6808","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6809","nonce":2129965200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6810","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6811","nonce":2129965200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6814","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6815","nonce":2129965200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6812","nonce":2129965200},{"type":"v1","addr":"172.21.15.31:6813","nonce":2129965200}]},"public_addr":"172.21.15.31:6809/2129965200","cluster_addr":"172.21.15.31:6811/2129965200","heartbeat_back_addr":"172.21.15.31:6815/2129965200","heartbeat_front_addr":"172.21.15.31:6813/2129965200","state":["exists","up"]},{"osd":6,"uuid":"c0b318df-6e83-4313-87bc-8d522ab34817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6816","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6817","nonce":2805350526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6818","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6819","nonce":2805350526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6822","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6823","nonce":2805350526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6820","nonce":2805350526},{"type":"v1","addr":"172.21.15.31:6821","nonce":2805350526}]},"public_addr":"172.21.15.31:6817/2805350526","cluster_addr":"172.21.15.31:6819/2805350526","heartbeat_back_addr":"172.21.15.31:6823/2805350526","heartbeat_front_addr":"172.21.15.31:6821/2805350526","state":["exists","up"]},{"osd":7,"uuid":"0cb97aa0-e5b2-4583-a665-c54670fa6d8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6824","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6825","nonce":121977008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6826","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6827","nonce":121977008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6830","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6831","nonce":121977008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.31:6828","nonce":121977008},{"type":"v1","addr":"172.21.15.31:6829","nonce":121977008}]},"public_addr":"172.21.15.31:6825/121977008","cluster_addr":"172.21.15.31:6827/121977008","heartbeat_back_addr":"172.21.15.31:6831/121977008","heartbeat_front_addr":"172.21.15.31:6829/121977008","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:33.646996+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:50:57.612531+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:21.810033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:51:47.753462+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:09.754493+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:33.863531+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:52:57.824927+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T23:53:22.657616+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/706006390":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6801/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/1509280396":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/2514861664":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6800/586451024":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/451441213":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/155433796":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6801/586451024":"2024-09-21T23:53:43.640164+0000","172.21.15.18:6800/1339690699":"2024-09-21T23:49:18.975789+0000","172.21.15.18:0/2751113824":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/2767829388":"2024-09-21T23:48:56.671898+0000","172.21.15.18:6800/2006779287":"2024-09-21T23:48:56.671898+0000","172.21.15.18:0/2320652463":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/513740309":"2024-09-21T23:53:43.640164+0000","172.21.15.18:0/3672252612":"2024-09-21T23:49:18.975789+0000","172.21.15.18:6801/2006779287":"2024-09-21T23:48:56.671898+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T23:54:09.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:09 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:09.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:09 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/628419323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:09.870 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:09 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:09.870 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:09 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/628419323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:10.045 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:09 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:10.045 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:09 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/628419323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T23:54:10.861 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-20T23:54:10.861 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-20T23:54:10.862 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-20T23:54:10.862 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-20T23:54:10.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-20T23:54:10.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-20T23:54:10.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-20T23:54:10.864 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-20T23:54:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:10 smithi018 ceph-mon[23740]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:10 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:11.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:10 smithi018 ceph-mon[28324]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:11.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:10 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:10 smithi031 ceph-mon[24497]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:10 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:11.292 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.301 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.327 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.328 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.330 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.331 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.331 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:11.356 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:13.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:12 smithi018 ceph-mon[23740]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:13.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:12 smithi018 ceph-mon[28324]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:13.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:12 smithi031 ceph-mon[24497]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:13.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:54:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:54:15.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:14 smithi018 ceph-mon[28324]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:15.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:14 smithi018 ceph-mon[23740]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:14 smithi031 ceph-mon[24497]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:16 smithi018 ceph-mon[28324]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:16 smithi018 ceph-mon[23740]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:16 smithi031 ceph-mon[24497]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:19.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:18 smithi018 ceph-mon[23740]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:19.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:18 smithi018 ceph-mon[28324]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:18 smithi031 ceph-mon[24497]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:19.345 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:19] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-09-20T23:54:21.020 INFO:teuthology.orchestra.run.smithi018.stdout:34359738414 2024-09-20T23:54:21.021 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[28324]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[28324]: Deploying daemon alertmanager.a on smithi018 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[23740]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:21.286 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:21 smithi018 ceph-mon[23740]: Deploying daemon alertmanager.a on smithi018 2024-09-20T23:54:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:21 smithi031 ceph-mon[24497]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:21 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:21 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:21.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:21 smithi031 ceph-mon[24497]: Deploying daemon alertmanager.a on smithi018 2024-09-20T23:54:22.162 INFO:teuthology.orchestra.run.smithi018.stdout:167503724567 2024-09-20T23:54:22.163 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-20T23:54:23.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:23 smithi018 ceph-mon[28324]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:23.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:23 smithi018 ceph-mon[23740]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:23 smithi031 ceph-mon[24497]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:23.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:54:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:54:24.963 INFO:teuthology.orchestra.run.smithi018.stdout:201863462931 2024-09-20T23:54:24.964 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-20T23:54:25.247 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:25 smithi018 ceph-mon[23740]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:25.247 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:25 smithi018 ceph-mon[28324]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:25.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:25 smithi031 ceph-mon[24497]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:25.929 INFO:teuthology.orchestra.run.smithi018.stdout:111669149728 2024-09-20T23:54:25.929 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-20T23:54:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:26 smithi018 ceph-mon[23740]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:26 smithi018 ceph-mon[28324]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:26 smithi031 ceph-mon[24497]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:27.116 INFO:teuthology.orchestra.run.smithi018.stdout:77309411366 2024-09-20T23:54:27.116 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-20T23:54:27.770 INFO:teuthology.orchestra.run.smithi018.stdout:55834574891 2024-09-20T23:54:27.771 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-20T23:54:28.656 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:28.657 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:28.657 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:28.657 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:28.660 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:29.015 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:28 smithi018 ceph-mon[28324]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:29.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:28 smithi018 ceph-mon[23740]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:29.056 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:28 smithi031 ceph-mon[24497]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:29.420 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:29] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2024-09-20T23:54:30.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:30 smithi018 ceph-mon[23740]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:30.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:30 smithi018 ceph-mon[28324]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:30 smithi031 ceph-mon[24497]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:31.792 INFO:teuthology.orchestra.run.smithi018.stdout:236223201294 2024-09-20T23:54:31.792 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-20T23:54:33.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:32 smithi018 ceph-mon[23740]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:33.039 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:32 smithi018 ceph-mon[28324]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:33.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:32 smithi031 ceph-mon[24497]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:33.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:54:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:54:34.727 INFO:teuthology.orchestra.run.smithi018.stdout:133143986205 2024-09-20T23:54:34.727 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-20T23:54:35.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:34 smithi018 ceph-mon[23740]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:35.116 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:34 smithi018 ceph-mon[28324]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:34 smithi031 ceph-mon[24497]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:36.528 INFO:teuthology.orchestra.run.smithi018.stdout:77309411368 2024-09-20T23:54:36.568 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:36 smithi018 systemd[1]: Starting Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:54:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:36 smithi018 ceph-mon[23740]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:36 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4218085287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T23:54:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:36 smithi018 ceph-mon[28324]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:36 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4218085287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T23:54:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:36 smithi031 ceph-mon[24497]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:36 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4218085287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T23:54:38.790 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411368 for osd.2 2024-09-20T23:54:38.790 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:38 smithi018 ceph-mon[23740]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:38 smithi018 ceph-mon[28324]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:38 smithi031 ceph-mon[24497]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:39.386 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:39.403 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:39] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2024-09-20T23:54:39.820 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:39.820 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:39.914 INFO:teuthology.orchestra.run.smithi018.stdout:55834574894 2024-09-20T23:54:41.021 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:40 smithi018 ceph-mon[28324]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:41.021 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:40 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/940994762' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T23:54:41.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:40 smithi018 ceph-mon[23740]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:41.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:40 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/940994762' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T23:54:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:40 smithi031 ceph-mon[24497]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:40 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/940994762' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T23:54:42.063 INFO:teuthology.orchestra.run.smithi018.stdout:111669149732 2024-09-20T23:54:42.693 INFO:teuthology.orchestra.run.smithi018.stdout:167503724571 2024-09-20T23:54:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[23740]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2886528260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T23:54:42.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3135918682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T23:54:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[28324]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2886528260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T23:54:42.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3135918682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T23:54:43.086 INFO:teuthology.orchestra.run.smithi018.stdout:34359738419 2024-09-20T23:54:43.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:42 smithi031 ceph-mon[24497]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:43.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2886528260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T23:54:43.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3135918682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T23:54:43.415 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574894 for osd.1 2024-09-20T23:54:43.416 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:43.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:54:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:54:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3346667389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T23:54:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:43.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3346667389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T23:54:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:44.174 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724571 for osd.5 2024-09-20T23:54:44.174 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:43 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3346667389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T23:54:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[23740]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[28324]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:44.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:45.222 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738419 for osd.0 2024-09-20T23:54:45.222 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:44 smithi031 ceph-mon[24497]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:44 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:54:45.631 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:45 smithi018 podman[46034]: 2024-09-20 23:54:45.244708622 +0000 UTC m=+3.553009919 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-20T23:54:45.658 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149732 for osd.3 2024-09-20T23:54:45.658 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:46.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:46 smithi018 ceph-mon[23740]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:46.912 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:46 smithi018 ceph-mon[28324]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:46 smithi031 ceph-mon[24497]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:47.515 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:47 smithi018 podman[46034]: 2024-09-20 23:54:47.297140995 +0000 UTC m=+5.605442292 volume create 8f18d6811b01098453f314ec989d9ff54c0020a5e43bad2f60f73d34f888fabb 2024-09-20T23:54:47.725 INFO:teuthology.orchestra.run.smithi018.stdout:201863462936 2024-09-20T23:54:47.777 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:47 smithi018 podman[46034]: 2024-09-20 23:54:47.707191783 +0000 UTC m=+6.015493079 container create 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:48.036 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:47 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1225950385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T23:54:48.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:47 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1225950385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T23:54:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:47 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1225950385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T23:54:48.330 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 podman[46034]: 2024-09-20 23:54:48.237707004 +0000 UTC m=+6.546008300 container init 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:48.330 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 podman[46034]: 2024-09-20 23:54:48.240865021 +0000 UTC m=+6.549166326 container start 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:54:48.330 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.251Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-20T23:54:48.330 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.251Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.253Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.254Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.279Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.279Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.282Z caller=main.go:518 msg=Listening address=:9093 2024-09-20T23:54:48.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:48.282Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-20T23:54:48.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 bash[46034]: 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 2024-09-20T23:54:48.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:48 smithi018 systemd[1]: Started Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:54:48.738 INFO:teuthology.orchestra.run.smithi018.stdout:133143986209 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/751450886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:49.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/751450886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T23:54:49.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:48 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462936 for osd.6 2024-09-20T23:54:49.211 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/751450886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T23:54:49.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:48 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:54:49.380 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:49] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-09-20T23:54:50.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:49 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:50.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:49 smithi018 ceph-mon[23740]: Deploying daemon grafana.a on smithi031 2024-09-20T23:54:50.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:49 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:50.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:49 smithi018 ceph-mon[28324]: Deploying daemon grafana.a on smithi031 2024-09-20T23:54:50.195 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986205 got 133143986209 for osd.4 2024-09-20T23:54:50.196 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:49 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T23:54:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:49 smithi031 ceph-mon[24497]: Deploying daemon grafana.a on smithi031 2024-09-20T23:54:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:50.254Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000362545s 2024-09-20T23:54:50.927 INFO:teuthology.orchestra.run.smithi018.stdout:236223201298 2024-09-20T23:54:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:50 smithi018 ceph-mon[28324]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:50 smithi018 ceph-mon[23740]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:50 smithi031 ceph-mon[24497]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:51.526 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-09-20T23:54:51.526 DEBUG:teuthology.parallel:result is None 2024-09-20T23:54:51.526 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-20T23:54:51.526 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T23:54:51.750 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:52.069 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:51 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/442557277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T23:54:52.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:51 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/442557277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T23:54:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:51 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/442557277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T23:54:53.022 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:52 smithi018 ceph-mon[23740]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:53.023 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:52 smithi018 ceph-mon[28324]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:53.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:52 smithi031 ceph-mon[24497]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:53.406 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:53.408 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2024-09-20T23:54:53.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:54:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:54:54.021 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-09-20T23:54:51.713510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3511269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002214"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T23:53:43.783164+0000","last_change":"2024-09-20T23:53:28.650434+0000","last_active":"2024-09-20T23:53:43.783164+0000","last_peered":"2024-09-20T23:53:43.783164+0000","last_clean":"2024-09-20T23:53:43.783164+0000","last_became_active":"2024-09-20T23:53:28.649975+0000","last_became_peered":"2024-09-20T23:53:28.649975+0000","last_unstale":"2024-09-20T23:53:43.783164+0000","last_undegraded":"2024-09-20T23:53:43.783164+0000","last_fullsized":"2024-09-20T23:53:43.783164+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T23:51:25.722067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T23:51:25.722067+0000","last_clean_scrub_stamp":"2024-09-20T23:51:25.722067+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:25:44.753615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60299999999999998}]},{"osd":1,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.57699999999999996}]},{"osd":2,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63}]},{"osd":3,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.69299999999999995}]},{"osd":4,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.40999999999999998}]},{"osd":5,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.42299999999999999}]},{"osd":6,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.50700000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.47799999999999998}]},{"osd":1,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.52700000000000002}]},{"osd":3,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.74099999999999999}]},{"osd":4,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.64800000000000002}]},{"osd":5,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.622},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.59399999999999997}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.67600000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.34399999999999997,"15min":0.317},"min":{"1min":0.217,"5min":0.184,"15min":0.184},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.22},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.33800000000000002},"min":{"1min":0.22800000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.52100000000000002,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.26800000000000002}]},{"osd":2,"last update":"Fri Sep 20 23:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.60899999999999999,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.31},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40200000000000002,"15min":0.38600000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56100000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.32100000000000001}]},{"osd":3,"last update":"Fri Sep 20 23:53:50 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.442,"15min":0.44800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.48799999999999999,"15min":0.49299999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.60399999999999998,"5min":0.746,"15min":0.746},"last":0.376}]},{"osd":4,"last update":"Fri Sep 20 23:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.74299999999999999,"5min":0.998,"15min":0.998},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78100000000000003,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.35599999999999998}]},{"osd":5,"last update":"Fri Sep 20 23:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77900000000000003,"5min":1.026,"15min":1.026},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.436}]},{"osd":7,"last update":"Fri Sep 20 23:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.41699999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.32200000000000001},"min":{"1min":0.25900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.54700000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.33900000000000002},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.215},"max":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.29099999999999998}]},{"osd":2,"last update":"Fri Sep 20 23:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.378,"15min":0.36699999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60399999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.308},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.38500000000000001},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.60199999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.41999999999999998}]},{"osd":3,"last update":"Fri Sep 20 23:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45100000000000001,"15min":0.45300000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.625,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64000000000000001,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.44600000000000001}]},{"osd":4,"last update":"Fri Sep 20 23:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78200000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.65000000000000002,"15min":0.65600000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.56000000000000005}]},{"osd":5,"last update":"Fri Sep 20 23:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.624,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.73099999999999998,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.5},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.60199999999999998,"15min":0.59999999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.63300000000000001}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.51200000000000001}]},{"osd":7,"last update":"Fri Sep 20 23:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.6120000000000001,"5min":2.6120000000000001,"15min":2.6120000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":3.548,"5min":3.548,"15min":3.548},"last":0.58499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.39800000000000002,"15min":0.39600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.496,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.38800000000000001,"15min":0.38400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.443,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.45500000000000002}]},{"osd":1,"last update":"Fri Sep 20 23:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41499999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.48999999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.51200000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Fri Sep 20 23:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.59999999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.442},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.48099999999999998,"15min":0.48299999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.61599999999999999,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.47399999999999998}]},{"osd":4,"last update":"Fri Sep 20 23:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.58699999999999997,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69999999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.61799999999999999,"15min":0.624},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78100000000000003,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59699999999999998}]},{"osd":5,"last update":"Fri Sep 20 23:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.627,"15min":0.63600000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82899999999999996,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.624},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.58599999999999997}]},{"osd":6,"last update":"Fri Sep 20 23:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53500000000000003}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.65000000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.51000000000000001,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.247},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.307}]},{"osd":1,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45400000000000001,"15min":0.45600000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.437,"15min":0.43099999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.28999999999999998}]},{"osd":2,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.45400000000000001,"15min":0.45900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.33000000000000002}]},{"osd":4,"last update":"Fri Sep 20 23:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60699999999999998,"15min":0.60799999999999998},"min":{"1min":0.443,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81599999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46100000000000002}]},{"osd":5,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.629,"15min":0.63100000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.51900000000000002}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.54200000000000004}]},{"osd":7,"last update":"Fri Sep 20 23:54:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.504},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.52900000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.56899999999999995,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69699999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69899999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.41599999999999998}]},{"osd":1,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.70699999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73899999999999999,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.437}]},{"osd":2,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.56200000000000006}]},{"osd":3,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71899999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56799999999999995,"15min":0.56899999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.66500000000000004}]},{"osd":5,"last update":"Fri Sep 20 23:54:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.40699999999999997,"15min":0.39400000000000002},"min":{"1min":0.35199999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.435,"15min":0.42099999999999999},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.69599999999999995}]},{"osd":6,"last update":"Fri Sep 20 23:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.62},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.54500000000000004}]},{"osd":7,"last update":"Fri Sep 20 23:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.68100000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.57399999999999995,"15min":0.58899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73399999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.55400000000000005,"15min":0.56799999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64500000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.67900000000000005}]},{"osd":1,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.55800000000000005,"15min":0.57199999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.59199999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.57399999999999995,"15min":0.58499999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.72499999999999998}]},{"osd":2,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.56000000000000005,"15min":0.56899999999999995},"min":{"1min":0.35299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70599999999999996,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69599999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60099999999999998}]},{"osd":3,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.56299999999999994,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.75,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.74099999999999999,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.626}]},{"osd":4,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80400000000000005,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.47899999999999998,"15min":0.47699999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.64900000000000002}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.76500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T23:54:54.023 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T23:54:54.237 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:55.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:54 smithi018 ceph-mon[28324]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:55.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:54 smithi018 ceph-mon[28324]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:55.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:54 smithi018 ceph-mon[23740]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:55.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:54 smithi018 ceph-mon[23740]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:54 smithi031 ceph-mon[24497]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:54 smithi031 ceph-mon[24497]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:56.039 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:56.041 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2024-09-20T23:54:56.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:56 smithi018 ceph-mon[23740]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:56.619 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:56 smithi018 ceph-mon[23740]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:56.619 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:56 smithi018 ceph-mon[28324]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:56.619 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:56 smithi018 ceph-mon[28324]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:56.646 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-09-20T23:54:55.714408+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3511269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002317"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T23:53:43.783164+0000","last_change":"2024-09-20T23:53:28.650434+0000","last_active":"2024-09-20T23:53:43.783164+0000","last_peered":"2024-09-20T23:53:43.783164+0000","last_clean":"2024-09-20T23:53:43.783164+0000","last_became_active":"2024-09-20T23:53:28.649975+0000","last_became_peered":"2024-09-20T23:53:28.649975+0000","last_unstale":"2024-09-20T23:53:43.783164+0000","last_undegraded":"2024-09-20T23:53:43.783164+0000","last_fullsized":"2024-09-20T23:53:43.783164+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T23:51:25.722067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T23:51:25.722067+0000","last_clean_scrub_stamp":"2024-09-20T23:51:25.722067+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:25:44.753615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.51100000000000001}]},{"osd":1,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.378},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53800000000000003}]},{"osd":2,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.41699999999999998}]},{"osd":3,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.499},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.44900000000000001}]},{"osd":4,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.58199999999999996}]},{"osd":5,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.624},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.60799999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.51800000000000002}]},{"osd":1,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.60099999999999998}]},{"osd":2,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.42599999999999999}]},{"osd":3,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.57199999999999995}]},{"osd":4,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.54500000000000004}]},{"osd":5,"last update":"Fri Sep 20 23:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.621}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.63},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.59499999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.34399999999999997,"15min":0.317},"min":{"1min":0.217,"5min":0.184,"15min":0.184},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.217},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.33800000000000002},"min":{"1min":0.22800000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.52100000000000002,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Fri Sep 20 23:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.60899999999999999,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40200000000000002,"15min":0.38600000000000001},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56100000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.27500000000000002}]},{"osd":3,"last update":"Fri Sep 20 23:54:52 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.42999999999999999,"15min":0.44400000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.502,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.253},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.47299999999999998,"15min":0.48799999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.52600000000000002,"5min":0.746,"15min":0.746},"last":0.308}]},{"osd":4,"last update":"Fri Sep 20 23:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.74299999999999999,"5min":0.998,"15min":0.998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78100000000000003,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.47799999999999998}]},{"osd":5,"last update":"Fri Sep 20 23:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77900000000000003,"5min":1.026,"15min":1.026},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.33500000000000002}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.40200000000000002}]},{"osd":7,"last update":"Fri Sep 20 23:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.45500000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.32200000000000001},"min":{"1min":0.25900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.54700000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.33900000000000002},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.215},"max":{"1min":0.56599999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.29499999999999998}]},{"osd":2,"last update":"Fri Sep 20 23:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.378,"15min":0.36699999999999999},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60399999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.38500000000000001},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.60199999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.35199999999999998}]},{"osd":3,"last update":"Fri Sep 20 23:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45100000000000001,"15min":0.45300000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.625,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64000000000000001,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.38300000000000001}]},{"osd":4,"last update":"Fri Sep 20 23:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78200000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.65000000000000002,"15min":0.65600000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.44600000000000001}]},{"osd":5,"last update":"Fri Sep 20 23:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.624,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.73099999999999998,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.60199999999999998,"15min":0.59999999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.53100000000000003}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.44}]},{"osd":7,"last update":"Fri Sep 20 23:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.6120000000000001,"5min":2.6120000000000001,"15min":2.6120000000000001},"last":0.502},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":3.548,"5min":3.548,"15min":3.548},"last":0.55400000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.39800000000000002,"15min":0.39600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.496,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.38800000000000001,"15min":0.38400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.443,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.439}]},{"osd":1,"last update":"Fri Sep 20 23:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41499999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.48999999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.51200000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Fri Sep 20 23:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.59999999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.48099999999999998,"15min":0.48299999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.61599999999999999,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.52200000000000002}]},{"osd":4,"last update":"Fri Sep 20 23:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.58699999999999997,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69999999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.61799999999999999,"15min":0.624},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78100000000000003,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.625}]},{"osd":5,"last update":"Fri Sep 20 23:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.627,"15min":0.63600000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82899999999999996,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Fri Sep 20 23:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.66800000000000004}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.751}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.51000000000000001,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.308}]},{"osd":1,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45400000000000001,"15min":0.45600000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.437,"15min":0.43099999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.29699999999999999}]},{"osd":2,"last update":"Fri Sep 20 23:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.45400000000000001,"15min":0.45900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.28299999999999997}]},{"osd":4,"last update":"Fri Sep 20 23:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60699999999999998,"15min":0.60799999999999998},"min":{"1min":0.443,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81599999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.40699999999999997}]},{"osd":5,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.629,"15min":0.63100000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.36799999999999999}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.376},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.47399999999999998}]},{"osd":7,"last update":"Fri Sep 20 23:54:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.505},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.46000000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.56899999999999995,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69699999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69899999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48099999999999998}]},{"osd":1,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.70699999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73899999999999999,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54000000000000004}]},{"osd":2,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.50800000000000001}]},{"osd":3,"last update":"Fri Sep 20 23:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71899999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56799999999999995,"15min":0.56899999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.56899999999999995}]},{"osd":5,"last update":"Fri Sep 20 23:54:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.40699999999999997,"15min":0.39400000000000002},"min":{"1min":0.35199999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.435,"15min":0.42099999999999999},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.623}]},{"osd":6,"last update":"Fri Sep 20 23:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.51800000000000002}]},{"osd":7,"last update":"Fri Sep 20 23:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.39000000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.57399999999999995,"15min":0.58899999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73399999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.55400000000000005,"15min":0.56799999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64500000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.41899999999999998}]},{"osd":1,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.55800000000000005,"15min":0.57199999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.59199999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.57399999999999995,"15min":0.58499999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.46899999999999997}]},{"osd":2,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.56000000000000005,"15min":0.56899999999999995},"min":{"1min":0.35299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70599999999999996,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69599999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.68899999999999995}]},{"osd":3,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.56299999999999994,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.75,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.74099999999999999,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.71599999999999997}]},{"osd":4,"last update":"Fri Sep 20 23:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80400000000000005,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.47899999999999998,"15min":0.47699999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.40500000000000003}]},{"osd":6,"last update":"Fri Sep 20 23:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.45300000000000001}]},{"osd":7,"last update":"Fri Sep 20 23:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.66800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T23:54:56.648 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-20T23:54:56.648 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-20T23:54:56.648 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-20T23:54:56.648 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- ceph health --format=json 2024-09-20T23:54:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:56 smithi031 ceph-mon[24497]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:56 smithi031 ceph-mon[24497]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T23:54:56.866 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/mon.a/config 2024-09-20T23:54:58.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:54:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:54:58.257Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002547094s 2024-09-20T23:54:58.718 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:54:58.718 INFO:teuthology.orchestra.run.smithi018.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-20T23:54:58.994 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:58 smithi031 ceph-mon[24497]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:58.994 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:54:58 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3402966845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T23:54:58.999 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:58 smithi018 ceph-mon[23740]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:58.999 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:54:58 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3402966845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T23:54:58.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:58 smithi018 ceph-mon[28324]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:54:58.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:54:58 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3402966845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T23:54:59.251 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:54:59 smithi031 systemd[1]: Starting Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:54:59.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:54:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:54:59] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-09-20T23:54:59.299 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-20T23:54:59.299 INFO:tasks.cephadm:Setup complete, yielding 2024-09-20T23:54:59.299 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T23:54:59.310 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-20T23:54:59.310 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-09-20T23:54:59.751 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:54:59 smithi031 podman[40608]: 2024-09-20 23:54:59.295118263 +0000 UTC m=+0.018296419 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-20T23:55:00.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:00 smithi031 podman[40608]: 2024-09-20 23:55:00.291399578 +0000 UTC m=+1.014577714 container create 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, io.buildah.version=1.24.2, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, version=8.5, name=ubi8) 2024-09-20T23:55:00.771 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:00 smithi018 ceph-mon[23740]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:01.049 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:00 smithi018 ceph-mon[28324]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:01.182 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:00 smithi031 ceph-mon[24497]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:01.843 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 podman[40608]: 2024-09-20 23:55:01.591586428 +0000 UTC m=+2.314764577 container init 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., release=236.1648460182) 2024-09-20T23:55:01.843 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 podman[40608]: 2024-09-20 23:55:01.595463042 +0000 UTC m=+2.318641178 container start 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, version=8.5, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, vcs-type=git, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-20T23:55:01.843 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-20T23:55:01.843 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-20T23:55:01.843 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-20T23:55:01.844 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-20T23:55:01.844 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-20T23:55:01.844 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-20T23:55:01.844 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="App mode production" logger=settings 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-09-20T23:55:01.845 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-09-20T23:55:01.846 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-09-20T23:55:01.847 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-09-20T23:55:01.848 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-09-20T23:55:01.849 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 bash[40608]: 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 2024-09-20T23:55:01.850 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-09-20T23:55:01.850 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 systemd[1]: Started Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:55:01.850 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-09-20T23:55:01.851 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-09-20T23:55:01.852 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-09-20T23:55:01.853 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-09-20T23:55:01.854 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-09-20T23:55:01.855 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-09-20T23:55:01.856 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-09-20T23:55:01.857 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-20T23:55:01.858 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-20T23:55:01.859 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T23:55:01.868 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-20T23:55:01.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-09-20T23:55:02.093 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-09-20T23:55:02.093 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-09-20T23:55:02.093 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-09-20T23:55:02.094 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-09-20T23:55:02.095 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-09-20T23:55:02.095 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-09-20T23:55:02.095 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-09-20T23:55:02.095 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-09-20T23:55:02.096 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-09-20T23:55:02.096 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-09-20T23:55:02.096 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-09-20T23:55:02.097 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-20T23:55:02.098 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-09-20T23:55:02.099 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-20T23:55:02.100 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-20T23:55:02.100 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-20T23:55:02.100 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-09-20T23:55:02.101 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-20T23:55:02.102 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-20T23:55:02.103 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-20T23:55:02.104 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-20T23:55:02.105 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-20T23:55:02.106 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-20T23:55:02.107 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-20T23:55:02.108 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-20T23:55:02.108 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-20T23:55:02.108 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:01+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-20T23:55:02.108 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-20T23:55:02.108 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-09-20T23:55:02.109 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-20T23:55:02.110 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-09-20T23:55:02.111 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-09-20T23:55:02.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-09-20T23:55:02.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-09-20T23:55:02.412 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1863742835' entity='client.admin' 2024-09-20T23:55:02.413 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:02.413 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:02.413 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:02.413 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-09-20T23:55:02.413 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-09-20T23:55:02.413 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-09-20T23:55:02.414 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-09-20T23:55:02.415 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=508.088722ms 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-20T23:55:02.416 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-20T23:55:02.417 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-20T23:55:02.417 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-20T23:55:02.417 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-20T23:55:02.417 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 20 23:55:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-20T23:55:02+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-20T23:55:02.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1863742835' entity='client.admin' 2024-09-20T23:55:02.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:02.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:02.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:02.492 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1863742835' entity='client.admin' 2024-09-20T23:55:02.492 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:02.492 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:02.492 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:02 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:03 smithi018 ceph-mon[23740]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:03 smithi018 ceph-mon[28324]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:03.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:03 smithi031 ceph-mon[24497]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:03.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:55:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:55:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:04 smithi018 ceph-mon[23740]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:04 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:04 smithi018 ceph-mon[28324]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:04 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:04 smithi031 ceph-mon[24497]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:04 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "99f02974-6ccc-4f37-8d01-4fa65b7c9bac", 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "r", 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout: "current_period": "62ed1dc7-c48d-4514-85b3-7a6d6e4ca07e", 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout: "epoch": 1 2024-09-20T23:55:05.814 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:55:06.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[28324]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T23:55:06.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-20T23:55:06.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[23740]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T23:55:06.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-20T23:55:06.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:05 smithi031 ceph-mon[24497]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T23:55:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:05 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-20T23:55:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:06.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-09-20T23:55:07.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:07.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-20T23:55:07.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T23:55:07.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:07.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[28324]: Reconfiguring daemon alertmanager.a on smithi018 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-20T23:55:07.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:06 smithi018 ceph-mon[23740]: Reconfiguring daemon alertmanager.a on smithi018 2024-09-20T23:55:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/462148402' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-20T23:55:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T23:55:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:07.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-20T23:55:07.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:06 smithi031 ceph-mon[24497]: Reconfiguring daemon alertmanager.a on smithi018 2024-09-20T23:55:08.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:07 smithi018 ceph-mon[23740]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T23:55:08.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:07 smithi018 ceph-mon[28324]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T23:55:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:07 smithi031 ceph-mon[24497]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T23:55:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:08 smithi018 ceph-mon[23740]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:08.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:08 smithi018 ceph-mon[28324]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:08 smithi031 ceph-mon[24497]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:09.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:55:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:09] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "1bc377f6-a49b-4ae2-8ca7-345b27942964", 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "default", 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "api_name": "default", 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "is_master": "true", 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "endpoints": [], 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "hostnames": [], 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "hostnames_s3website": [], 2024-09-20T23:55:10.259 INFO:teuthology.orchestra.run.smithi018.stdout: "master_zone": "", 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "zones": [], 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "placement_targets": [], 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "default_placement": "", 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_id": "99f02974-6ccc-4f37-8d01-4fa65b7c9bac", 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "sync_policy": { 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: "groups": [] 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:10.260 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:55:10.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:10 smithi018 ceph-mon[23740]: pgmap v49: 33 pgs: 22 active+clean, 11 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:10.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:10 smithi018 ceph-mon[28324]: pgmap v49: 33 pgs: 22 active+clean, 11 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:10.901 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-09-20T23:55:11.237 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:10 smithi031 ceph-mon[24497]: pgmap v49: 33 pgs: 22 active+clean, 11 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:11.637 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:11 smithi018 systemd[1]: Stopping Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:55:11.637 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:11 smithi018 bash[47929]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager.a" found: no such container 2024-09-20T23:55:11.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[46293]: level=info ts=2024-09-20T23:55:11.635Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-20T23:55:12.590 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 podman[47940]: 2024-09-20 23:55:12.285790405 +0000 UTC m=+0.757113640 container died 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:12.842 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:12 smithi018 ceph-mon[23740]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-20T23:55:12.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:12 smithi018 ceph-mon[28324]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-20T23:55:12.842 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 podman[47940]: 2024-09-20 23:55:12.589704792 +0000 UTC m=+1.061028059 container cleanup 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:12.843 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 bash[47940]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a 2024-09-20T23:55:12.843 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 podman[47966]: 2024-09-20 23:55:12.765673604 +0000 UTC m=+1.104873875 container remove 367bde36d60064493d448148761ad0f4f8cb1ecfec807a389fc5f3fb71a93f37 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:13.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:12 smithi031 ceph-mon[24497]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-20T23:55:13.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 podman[47966]: 2024-09-20 23:55:12.841500533 +0000 UTC m=+1.180700804 volume remove 8f18d6811b01098453f314ec989d9ff54c0020a5e43bad2f60f73d34f888fabb 2024-09-20T23:55:13.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 bash[47999]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager.a" found: no such container 2024-09-20T23:55:13.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-20T23:55:13.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 systemd[1]: Stopped Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:55:13.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:12 smithi018 systemd[1]: Starting Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:55:13.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:55:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "145118ec-b3c1-43de-9402-afb744d6003d", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "z", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "domain_root": "z.rgw.meta:root", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "control_pool": "z.rgw.control", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "gc_pool": "z.rgw.log:gc", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "lc_pool": "z.rgw.log:lc", 2024-09-20T23:55:13.881 INFO:teuthology.orchestra.run.smithi018.stdout: "log_pool": "z.rgw.log", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "roles_pool": "z.rgw.meta:roles", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "otp_pool": "z.rgw.otp", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "system_key": { 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "access_key": "", 2024-09-20T23:55:13.882 INFO:teuthology.orchestra.run.smithi018.stdout: "secret_key": "" 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "placement_pools": [ 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "key": "default-placement", 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "val": { 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "index_pool": "z.rgw.buckets.index", 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "storage_classes": { 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "STANDARD": { 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "data_pool": "z.rgw.buckets.data" 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: "index_type": 0 2024-09-20T23:55:13.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:13.884 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:13.884 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-20T23:55:13.884 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_id": "99f02974-6ccc-4f37-8d01-4fa65b7c9bac", 2024-09-20T23:55:13.884 INFO:teuthology.orchestra.run.smithi018.stdout: "notif_pool": "z.rgw.log:notif" 2024-09-20T23:55:13.884 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:55:14.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-09-20T23:55:14.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:14 smithi018 ceph-mon[23740]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:14.870 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:14 smithi018 ceph-mon[28324]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:14.871 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:14 smithi018 podman[48190]: 2024-09-20 23:55:14.665956304 +0000 UTC m=+0.027019910 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-20T23:55:14.871 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:14 smithi018 podman[48190]: 2024-09-20 23:55:14.785793307 +0000 UTC m=+0.146856908 volume create 0c4bbed7491e059da659b0804639f1504fdde5f3001d52a53607573c3b9b7be7 2024-09-20T23:55:14.871 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:14 smithi018 podman[48190]: 2024-09-20 23:55:14.869757396 +0000 UTC m=+0.230821031 container create 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:14 smithi031 ceph-mon[24497]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 podman[48190]: 2024-09-20 23:55:15.357965081 +0000 UTC m=+0.719028685 container init 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 podman[48190]: 2024-09-20 23:55:15.361466573 +0000 UTC m=+0.722530170 container start 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.373Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.373Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.375Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2024-09-20T23:55:15.403 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.376Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-20T23:55:15.404 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.402Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-20T23:55:15.404 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.402Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-20T23:55:15.660 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.404Z caller=main.go:518 msg=Listening address=:9093 2024-09-20T23:55:15.660 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:15.405Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-20T23:55:15.660 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 bash[48190]: 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e 2024-09-20T23:55:15.660 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:15 smithi018 systemd[1]: Started Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:55:16.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:16.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[28324]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-20T23:55:16.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[28324]: Reconfiguring daemon prometheus.a on smithi031 2024-09-20T23:55:16.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[28324]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 720 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:16.659 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:16.660 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[23740]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-20T23:55:16.660 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[23740]: Reconfiguring daemon prometheus.a on smithi031 2024-09-20T23:55:16.660 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:16 smithi018 ceph-mon[23740]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 720 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:16 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:16.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:16 smithi031 ceph-mon[24497]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-20T23:55:16.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:16 smithi031 ceph-mon[24497]: Reconfiguring daemon prometheus.a on smithi031 2024-09-20T23:55:16.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:16 smithi031 ceph-mon[24497]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 720 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:17.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:17.376Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000418038s 2024-09-20T23:55:18.705 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:18 smithi031 ceph-mon[24497]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T23:55:18.705 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:18 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-20T23:55:18.705 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:18 smithi031 ceph-mon[24497]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[28324]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[28324]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[23740]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-20T23:55:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:18 smithi018 ceph-mon[23740]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-09-20T23:55:19.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:18 smithi031 podman[41307]: 2024-09-20 23:55:18.935347738 +0000 UTC m=+0.134840635 container died 3d3f71baed2e452701a583c13151b8ed71eca5ccbd4f45c93338eee46b245f4e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:19.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 podman[41307]: 2024-09-20 23:55:19.315241253 +0000 UTC m=+0.514734139 container cleanup 3d3f71baed2e452701a583c13151b8ed71eca5ccbd4f45c93338eee46b245f4e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:19.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 bash[41307]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-20T23:55:19.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 bash[41335]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus.a" found: no such container 2024-09-20T23:55:19.843 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 podman[41318]: 2024-09-20 23:55:19.549474719 +0000 UTC m=+0.614787289 container remove 3d3f71baed2e452701a583c13151b8ed71eca5ccbd4f45c93338eee46b245f4e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:19.843 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-20T23:55:19.843 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 systemd[1]: Stopped Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:55:19.843 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 systemd[1]: Starting Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:55:19.843 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 podman[41399]: 2024-09-20 23:55:19.697350226 +0000 UTC m=+0.018508305 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-09-20T23:55:19.844 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:19 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-20T23:55:19.844 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:19 smithi031 ceph-mon[24497]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T23:55:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:19 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-20T23:55:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:19 smithi018 ceph-mon[23740]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T23:55:19.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:19 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-20T23:55:19.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:19 smithi018 ceph-mon[28324]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T23:55:20.097 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:19 smithi031 podman[41399]: 2024-09-20 23:55:19.842234137 +0000 UTC m=+0.163392258 container create ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:20.097 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 podman[41399]: 2024-09-20 23:55:20.069152081 +0000 UTC m=+0.390310158 container init ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:20.097 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 podman[41399]: 2024-09-20 23:55:20.072058434 +0000 UTC m=+0.393216512 container start ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-20T23:55:20.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-20T23:55:20.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-09-20T23:55:20.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-09-20T23:55:20.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:518 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi031 (none))" 2024-09-20T23:55:20.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.096Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.098Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.098Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.099Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.100Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.100Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.583µs 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.100Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 bash[41399]: ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 systemd[1]: Started Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.315Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.315Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-20T23:55:20.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.315Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.555µs wal_replay_duration=215.209519ms total_replay_duration=215.249097ms 2024-09-20T23:55:20.504 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.317Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-09-20T23:55:20.504 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.317Z caller=main.go:947 level=info msg="TSDB started" 2024-09-20T23:55:20.504 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.317Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-20T23:55:20.504 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.335Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.9514ms db_storage=812ns remote_storage=1.519µs web_handler=384ns query_engine=601ns scrape=506.642µs scrape_sd=25.839µs notify=20.369µs notify_sd=6.278µs rules=17.156334ms 2024-09-20T23:55:20.504 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-20T23:55:20.335Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-09-20T23:55:20.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T23:55:20.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: pgmap v57: 97 pgs: 24 creating+peering, 60 active+clean, 13 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.916 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[28324]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T23:55:20.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: pgmap v57: 97 pgs: 24 creating+peering, 60 active+clean, 13 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-20T23:55:20.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:20 smithi018 ceph-mon[23740]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T23:55:20.959 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T23:55:20.959 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-20T23:55:20.959 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: pgmap v57: 97 pgs: 24 creating+peering, 60 active+clean, 13 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-20T23:55:20.959 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.959 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.31:9095"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.960 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.31:3000"}]: dispatch 2024-09-20T23:55:20.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:20.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:20.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:20.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3501881354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-20T23:55:20.961 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:20 smithi031 ceph-mon[24497]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T23:55:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[23740]: pgmap v59: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[23740]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T23:55:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-20T23:55:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[28324]: pgmap v59: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[28324]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T23:55:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:22 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-20T23:55:23.223 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:22 smithi031 ceph-mon[24497]: pgmap v59: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:23.224 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:22 smithi031 ceph-mon[24497]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T23:55:23.224 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:22 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-20T23:55:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-20T23:55:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T23:55:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-20T23:55:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:23 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:24.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:23 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:24.912 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:55:24.912 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "fd65ec71-8225-403b-b6f5-c26aad7aa8eb", 2024-09-20T23:55:24.912 INFO:teuthology.orchestra.run.smithi018.stdout: "epoch": 1, 2024-09-20T23:55:24.912 INFO:teuthology.orchestra.run.smithi018.stdout: "predecessor_uuid": "62ed1dc7-c48d-4514-85b3-7a6d6e4ca07e", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "sync_status": [], 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "period_map": { 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "fd65ec71-8225-403b-b6f5-c26aad7aa8eb", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "zonegroups": [ 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "1bc377f6-a49b-4ae2-8ca7-345b27942964", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "default", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "api_name": "default", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "is_master": "true", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "endpoints": [], 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "hostnames": [], 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "hostnames_s3website": [], 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "master_zone": "145118ec-b3c1-43de-9402-afb744d6003d", 2024-09-20T23:55:24.913 INFO:teuthology.orchestra.run.smithi018.stdout: "zones": [ 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "id": "145118ec-b3c1-43de-9402-afb744d6003d", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "z", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "endpoints": [], 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "log_meta": "false", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "log_data": "false", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "bucket_index_max_shards": 11, 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "read_only": "false", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "tier_type": "", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "sync_from_all": "true", 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "sync_from": [], 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: "redirect_zone": "" 2024-09-20T23:55:24.914 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "placement_targets": [ 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "default-placement", 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "tags": [], 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "storage_classes": [ 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "STANDARD" 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "default_placement": "default-placement", 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_id": "99f02974-6ccc-4f37-8d01-4fa65b7c9bac", 2024-09-20T23:55:24.915 INFO:teuthology.orchestra.run.smithi018.stdout: "sync_policy": { 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: "groups": [] 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: "short_zone_ids": [ 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: "key": "145118ec-b3c1-43de-9402-afb744d6003d", 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: "val": 1292862484 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.916 INFO:teuthology.orchestra.run.smithi018.stdout: "master_zonegroup": "1bc377f6-a49b-4ae2-8ca7-345b27942964", 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "master_zone": "145118ec-b3c1-43de-9402-afb744d6003d", 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "period_config": { 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "bucket_quota": { 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "enabled": false, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "check_on_raw": false, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "max_size": -1, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "max_size_kb": 0, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "max_objects": -1 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "user_quota": { 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "enabled": false, 2024-09-20T23:55:24.917 INFO:teuthology.orchestra.run.smithi018.stdout: "check_on_raw": false, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_size": -1, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_size_kb": 0, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_objects": -1 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "user_ratelimit": { 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_ops": 0, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_ops": 0, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_bytes": 0, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_bytes": 0, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "enabled": false 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "bucket_ratelimit": { 2024-09-20T23:55:24.918 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_ops": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_ops": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_bytes": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_bytes": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "enabled": false 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "anonymous_ratelimit": { 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_ops": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_ops": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_read_bytes": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "max_write_bytes": 0, 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: "enabled": false 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:55:24.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:55:24.920 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_id": "99f02974-6ccc-4f37-8d01-4fa65b7c9bac", 2024-09-20T23:55:24.920 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_name": "r", 2024-09-20T23:55:24.920 INFO:teuthology.orchestra.run.smithi018.stdout: "realm_epoch": 2 2024-09-20T23:55:24.920 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:55:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[23740]: pgmap v62: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-20T23:55:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[23740]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T23:55:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-20T23:55:25.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[28324]: pgmap v62: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:25.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-20T23:55:25.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[28324]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T23:55:25.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:24 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-20T23:55:25.238 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:24 smithi031 ceph-mon[24497]: pgmap v62: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 5 op/s 2024-09-20T23:55:25.238 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:24 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-20T23:55:25.238 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:24 smithi031 ceph-mon[24497]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T23:55:25.238 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:24 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-20T23:55:25.664 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:25 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-20T23:55:25.379Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002841108s 2024-09-20T23:55:25.693 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-09-20T23:55:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:25 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-20T23:55:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:25 smithi018 ceph-mon[23740]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T23:55:25.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:25 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-20T23:55:25.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:25 smithi018 ceph-mon[28324]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T23:55:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:25 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2709783096' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-20T23:55:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:25 smithi031 ceph-mon[24497]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T23:55:27.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:26 smithi018 ceph-mon[23740]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-20T23:55:27.054 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:26 smithi018 ceph-mon[28324]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-20T23:55:27.207 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:26 smithi031 ceph-mon[24497]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-09-20T23:55:27.744 INFO:teuthology.orchestra.run.smithi018.stdout:Scheduled rgw.foo update... 2024-09-20T23:55:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:28 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:28.503 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:28.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:28 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:28.530 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph osd pool create foo' 2024-09-20T23:55:29.415 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:29] "GET /metrics HTTP/1.1" 200 196826 "" "Prometheus/2.33.4" 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[23740]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 174 B/s rd, 348 B/s wr, 1 op/s 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[23740]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[23740]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[23740]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[23740]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[28324]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 174 B/s rd, 348 B/s wr, 1 op/s 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[28324]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:29.416 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[28324]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[28324]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:29 smithi018 ceph-mon[28324]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-20T23:55:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:29 smithi031 ceph-mon[24497]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 174 B/s rd, 348 B/s wr, 1 op/s 2024-09-20T23:55:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:29 smithi031 ceph-mon[24497]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:29.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:29 smithi031 ceph-mon[24497]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:29 smithi031 ceph-mon[24497]: Saving service rgw.foo spec with placement count:2 2024-09-20T23:55:29.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:29 smithi031 ceph-mon[24497]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-20T23:55:31.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:31 smithi018 ceph-mon[23740]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-20T23:55:31.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:31 smithi018 ceph-mon[28324]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-20T23:55:31.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:31 smithi031 ceph-mon[24497]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-20T23:55:32.247 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'foo' created 2024-09-20T23:55:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:32 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-20T23:55:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:32 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:32 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:32 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:32 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T23:55:32.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:32 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T23:55:33.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'rbd pool init foo' 2024-09-20T23:55:33.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T23:55:33.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:33 smithi031 ceph-mon[24497]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T23:55:33.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[23740]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2602043248' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T23:55:33.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:33 smithi018 ceph-mon[28324]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-20T23:55:34.470 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:34 smithi031 ceph-mon[24497]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T23:55:34.579 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:34 smithi018 ceph-mon[28324]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T23:55:34.580 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:34 smithi018 ceph-mon[23740]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T23:55:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-20T23:55:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: osdmap e74: 8 total, 8 up, 8 in 2024-09-20T23:55:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:35.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:35.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:35 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: osdmap e74: 8 total, 8 up, 8 in 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: osdmap e74: 8 total, 8 up, 8 in 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:35 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-20T23:55:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:36 smithi031 ceph-mon[24497]: pgmap v73: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 5.7 KiB/s wr, 192 op/s 2024-09-20T23:55:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:36 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-20T23:55:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:36 smithi031 ceph-mon[24497]: osdmap e75: 8 total, 8 up, 8 in 2024-09-20T23:55:37.040 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[28324]: pgmap v73: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 5.7 KiB/s wr, 192 op/s 2024-09-20T23:55:37.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-20T23:55:37.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[28324]: osdmap e75: 8 total, 8 up, 8 in 2024-09-20T23:55:37.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[23740]: pgmap v73: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 5.7 KiB/s wr, 192 op/s 2024-09-20T23:55:37.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/400076962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-20T23:55:37.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:36 smithi018 ceph-mon[23740]: osdmap e75: 8 total, 8 up, 8 in 2024-09-20T23:55:38.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:37 smithi018 ceph-mon[28324]: osdmap e76: 8 total, 8 up, 8 in 2024-09-20T23:55:38.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:37 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:38.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:37 smithi018 ceph-mon[23740]: osdmap e76: 8 total, 8 up, 8 in 2024-09-20T23:55:38.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:37 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:38.090 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:37 smithi031 ceph-mon[24497]: osdmap e76: 8 total, 8 up, 8 in 2024-09-20T23:55:38.091 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:37 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:38.451 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch apply iscsi foo u p' 2024-09-20T23:55:39.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 7.6 KiB/s wr, 257 op/s 2024-09-20T23:55:39.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 7.6 KiB/s wr, 257 op/s 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:39.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:38 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 7.6 KiB/s wr, 257 op/s 2024-09-20T23:55:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T23:55:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:39.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:39.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:38 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:39.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:39] "GET /metrics HTTP/1.1" 200 196826 "" "Prometheus/2.33.4" 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[23740]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[23740]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[28324]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[28324]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:39 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:39 smithi031 ceph-mon[24497]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:39 smithi031 ceph-mon[24497]: Checking dashboard <-> RGW credentials 2024-09-20T23:55:40.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:39 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:39 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:40.537 INFO:teuthology.orchestra.run.smithi018.stdout:Scheduled iscsi.foo update... 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 6.9 KiB/s wr, 401 op/s 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 6.9 KiB/s wr, 401 op/s 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-20T23:55:41.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:40 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:41.234 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 6.9 KiB/s wr, 401 op/s 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-20T23:55:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-20T23:55:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:40 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[28324]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[28324]: Saving service iscsi.foo spec with placement count:1 2024-09-20T23:55:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[28324]: Checking pool "foo" exists for service iscsi.foo 2024-09-20T23:55:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[28324]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-20T23:55:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[23740]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[23740]: Saving service iscsi.foo spec with placement count:1 2024-09-20T23:55:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[23740]: Checking pool "foo" exists for service iscsi.foo 2024-09-20T23:55:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:41 smithi018 ceph-mon[23740]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-20T23:55:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:41 smithi031 ceph-mon[24497]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:55:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:41 smithi031 ceph-mon[24497]: Saving service iscsi.foo spec with placement count:1 2024-09-20T23:55:42.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:41 smithi031 ceph-mon[24497]: Checking pool "foo" exists for service iscsi.foo 2024-09-20T23:55:42.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:41 smithi031 ceph-mon[24497]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-20T23:55:43.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:42 smithi018 ceph-mon[23740]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 687 B/s wr, 175 op/s 2024-09-20T23:55:43.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:42 smithi018 ceph-mon[23740]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-20T23:55:43.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:42 smithi018 ceph-mon[28324]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 687 B/s wr, 175 op/s 2024-09-20T23:55:43.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:42 smithi018 ceph-mon[28324]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-20T23:55:43.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:55:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:42 smithi031 ceph-mon[24497]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 687 B/s wr, 175 op/s 2024-09-20T23:55:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:42 smithi031 ceph-mon[24497]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-20T23:55:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T23:55:44.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:55:44.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:45.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 588 B/s wr, 150 op/s 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[28324]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 588 B/s wr, 150 op/s 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-20T23:55:45.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-20T23:55:45.224 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-20T23:55:45.224 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:44 smithi018 ceph-mon[23740]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 588 B/s wr, 150 op/s 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-20T23:55:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:44 smithi031 ceph-mon[24497]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T23:55:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T23:55:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:46.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1477109457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-20T23:55:46.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3332823575' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:46.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:45 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1477109457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-20T23:55:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3332823575' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1477109457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3332823575' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:45 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]: dispatch 2024-09-20T23:55:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:46 smithi031 ceph-mon[24497]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:46 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]': finished 2024-09-20T23:55:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:46 smithi031 ceph-mon[24497]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T23:55:47.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:46 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]: dispatch 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[23740]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]': finished 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[23740]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]: dispatch 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[28324]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:47.411 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/706006390"}]': finished 2024-09-20T23:55:47.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[28324]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T23:55:47.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:46 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]: dispatch 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]': finished 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[23740]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[23740]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2798651962' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:48.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]': finished 2024-09-20T23:55:48.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[28324]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T23:55:48.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[28324]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:48.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2798651962' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:48.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:48 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:48 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2843471219' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1339690699"}]': finished 2024-09-20T23:55:49.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:48 smithi031 ceph-mon[24497]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T23:55:49.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:48 smithi031 ceph-mon[24497]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:49.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:48 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2798651962' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:49.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:48 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]: dispatch 2024-09-20T23:55:49.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:49] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]': finished 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2044079809' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]': finished 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:49.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2044079809' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:49.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:49 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1509280396"}]': finished 2024-09-20T23:55:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:55:50.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:55:50.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:55:50.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2044079809' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:50.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:49 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]: dispatch 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[23740]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 215 B/s wr, 2 op/s; 164 B/s, 0 objects/s recovering 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]': finished 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[23740]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/973480311' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[28324]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 215 B/s wr, 2 op/s; 164 B/s, 0 objects/s recovering 2024-09-20T23:55:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]': finished 2024-09-20T23:55:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[28324]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T23:55:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/973480311' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:50 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:50 smithi031 ceph-mon[24497]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 215 B/s wr, 2 op/s; 164 B/s, 0 objects/s recovering 2024-09-20T23:55:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:50 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2514861664"}]': finished 2024-09-20T23:55:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:50 smithi031 ceph-mon[24497]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T23:55:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:50 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/973480311' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:51.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:50 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]: dispatch 2024-09-20T23:55:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]': finished 2024-09-20T23:55:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[23740]: osdmap e85: 8 total, 8 up, 8 in 2024-09-20T23:55:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]: dispatch 2024-09-20T23:55:52.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]': finished 2024-09-20T23:55:52.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[28324]: osdmap e85: 8 total, 8 up, 8 in 2024-09-20T23:55:52.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:51 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]: dispatch 2024-09-20T23:55:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:51 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/586451024"}]': finished 2024-09-20T23:55:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:51 smithi031 ceph-mon[24497]: osdmap e85: 8 total, 8 up, 8 in 2024-09-20T23:55:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:51 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]: dispatch 2024-09-20T23:55:53.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:52 smithi031 ceph-mon[24497]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 247 B/s wr, 3 op/s; 189 B/s, 0 objects/s recovering 2024-09-20T23:55:53.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:52 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]': finished 2024-09-20T23:55:53.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:52 smithi031 ceph-mon[24497]: osdmap e86: 8 total, 8 up, 8 in 2024-09-20T23:55:53.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:52 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2158204648' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:52 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[23740]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 247 B/s wr, 3 op/s; 189 B/s, 0 objects/s recovering 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]': finished 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[23740]: osdmap e86: 8 total, 8 up, 8 in 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2158204648' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[28324]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 247 B/s wr, 3 op/s; 189 B/s, 0 objects/s recovering 2024-09-20T23:55:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2822509875' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/451441213"}]': finished 2024-09-20T23:55:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[28324]: osdmap e86: 8 total, 8 up, 8 in 2024-09-20T23:55:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2158204648' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:52 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]: dispatch 2024-09-20T23:55:53.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:55:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:55:53.823 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-20T23:55:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-20T23:55:53.823 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-20T23:55:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-20T23:55:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]': finished 2024-09-20T23:55:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[23740]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T23:55:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/406083435' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:54.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]': finished 2024-09-20T23:55:54.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[28324]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T23:55:54.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/406083435' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:54.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:53 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:53 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/155433796"}]': finished 2024-09-20T23:55:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:53 smithi031 ceph-mon[24497]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T23:55:54.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:53 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/406083435' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:54.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:53 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]: dispatch 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[28324]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]': finished 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[28324]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]: dispatch 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[23740]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]': finished 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[23740]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T23:55:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:54 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]: dispatch 2024-09-20T23:55:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:54 smithi031 ceph-mon[24497]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:55:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:54 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/586451024"}]': finished 2024-09-20T23:55:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:54 smithi031 ceph-mon[24497]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T23:55:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:54 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]: dispatch 2024-09-20T23:55:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]': finished 2024-09-20T23:55:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[23740]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T23:55:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/726857789' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]': finished 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[28324]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/726857789' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:55 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:55:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1166747633' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1339690699"}]': finished 2024-09-20T23:55:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:55 smithi031 ceph-mon[24497]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T23:55:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/726857789' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:55 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]: dispatch 2024-09-20T23:55:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:55 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[23740]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]': finished 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[23740]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1675359032' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[28324]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]': finished 2024-09-20T23:55:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[28324]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T23:55:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1675359032' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:56 smithi031 ceph-mon[24497]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2751113824"}]': finished 2024-09-20T23:55:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:56 smithi031 ceph-mon[24497]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T23:55:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:56 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1675359032' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]: dispatch 2024-09-20T23:55:58.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:57 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]': finished 2024-09-20T23:55:58.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:57 smithi031 ceph-mon[24497]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T23:55:58.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:57 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3744757714' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:58.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:57 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]': finished 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[23740]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3744757714' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2767829388"}]': finished 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[28324]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3744757714' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:57 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]: dispatch 2024-09-20T23:55:59.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:55:59] "GET /metrics HTTP/1.1" 200 208069 "" "Prometheus/2.33.4" 2024-09-20T23:55:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[28324]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]': finished 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[28324]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]: dispatch 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[23740]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]': finished 2024-09-20T23:55:59.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[23740]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T23:55:59.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:58 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]: dispatch 2024-09-20T23:55:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:58 smithi031 ceph-mon[24497]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:55:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:58 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/2006779287"}]': finished 2024-09-20T23:55:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:58 smithi031 ceph-mon[24497]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T23:55:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:58 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]: dispatch 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]': finished 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[23740]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2569223460' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]': finished 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[28324]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T23:56:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2569223460' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:00.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:55:59 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:59 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1073601171' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2320652463"}]': finished 2024-09-20T23:56:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:59 smithi031 ceph-mon[24497]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T23:56:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:59 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2569223460' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:55:59 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]: dispatch 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[23740]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]': finished 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[23740]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4282295621' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[28324]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]': finished 2024-09-20T23:56:01.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[28324]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T23:56:01.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4282295621' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:01.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:00 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:00 smithi031 ceph-mon[24497]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:00 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/513740309"}]': finished 2024-09-20T23:56:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:00 smithi031 ceph-mon[24497]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T23:56:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:00 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4282295621' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:01.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:00 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]: dispatch 2024-09-20T23:56:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]': finished 2024-09-20T23:56:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[23740]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T23:56:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]: dispatch 2024-09-20T23:56:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]': finished 2024-09-20T23:56:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[28324]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T23:56:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:01 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]: dispatch 2024-09-20T23:56:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:01 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3672252612"}]': finished 2024-09-20T23:56:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:01 smithi031 ceph-mon[24497]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T23:56:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:01 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]: dispatch 2024-09-20T23:56:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[23740]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]': finished 2024-09-20T23:56:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[23740]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T23:56:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[28324]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]': finished 2024-09-20T23:56:03.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:02 smithi018 ceph-mon[28324]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T23:56:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:02 smithi031 ceph-mon[24497]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:02 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/527200035' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/2006779287"}]': finished 2024-09-20T23:56:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:02 smithi031 ceph-mon[24497]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T23:56:03.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 20 23:56:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-20T23:56:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-20T23:56:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:04 smithi018 ceph-mon[23740]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-20T23:56:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:04 smithi018 ceph-mon[28324]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-20T23:56:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:04 smithi031 ceph-mon[24497]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-20T23:56:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:05 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:05 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:05 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:06 smithi018 ceph-mon[23740]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:06 smithi018 ceph-mon[28324]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:06 smithi031 ceph-mon[24497]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:08 smithi018 ceph-mon[23740]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-09-20T23:56:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:08 smithi018 ceph-mon[28324]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-09-20T23:56:09.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:09] "GET /metrics HTTP/1.1" 200 208069 "" "Prometheus/2.33.4" 2024-09-20T23:56:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:08 smithi031 ceph-mon[24497]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-09-20T23:56:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:10 smithi018 ceph-mon[23740]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-20T23:56:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:10 smithi018 ceph-mon[28324]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-20T23:56:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:10 smithi031 ceph-mon[24497]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-20T23:56:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:12 smithi018 ceph-mon[23740]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-20T23:56:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:12 smithi018 ceph-mon[28324]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-20T23:56:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:12 smithi031 ceph-mon[24497]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-20T23:56:13.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:14 smithi018 ceph-mon[23740]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-20T23:56:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:14 smithi018 ceph-mon[28324]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-20T23:56:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:14 smithi031 ceph-mon[24497]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-20T23:56:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:15 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:15 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:15 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:16 smithi018 ceph-mon[28324]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:16 smithi018 ceph-mon[23740]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:16 smithi031 ceph-mon[24497]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:18 smithi018 ceph-mon[23740]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:18 smithi018 ceph-mon[28324]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:19.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:19] "GET /metrics HTTP/1.1" 200 208049 "" "Prometheus/2.33.4" 2024-09-20T23:56:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:18 smithi031 ceph-mon[24497]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:20 smithi018 ceph-mon[23740]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:20 smithi018 ceph-mon[28324]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:20 smithi031 ceph-mon[24497]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:22 smithi018 ceph-mon[23740]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:22 smithi018 ceph-mon[28324]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:22 smithi031 ceph-mon[24497]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:23.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:24 smithi018 ceph-mon[23740]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:24 smithi018 ceph-mon[28324]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:24 smithi031 ceph-mon[24497]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:25 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:25 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:25 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:26 smithi018 ceph-mon[28324]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:26 smithi018 ceph-mon[23740]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:26 smithi031 ceph-mon[24497]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:28 smithi018 ceph-mon[23740]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:28 smithi018 ceph-mon[28324]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:29.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:29] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-09-20T23:56:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:28 smithi031 ceph-mon[24497]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:30 smithi018 ceph-mon[23740]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:30 smithi018 ceph-mon[28324]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:30 smithi031 ceph-mon[24497]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:32 smithi018 ceph-mon[23740]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:32 smithi018 ceph-mon[28324]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:32 smithi031 ceph-mon[24497]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:33.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:34 smithi018 ceph-mon[28324]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:34 smithi018 ceph-mon[23740]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:34 smithi031 ceph-mon[24497]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:35 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:35 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:35 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:36 smithi018 ceph-mon[23740]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:36 smithi018 ceph-mon[28324]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:36 smithi031 ceph-mon[24497]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:38 smithi018 ceph-mon[23740]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:38 smithi018 ceph-mon[28324]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:39.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:39] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-09-20T23:56:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:38 smithi031 ceph-mon[24497]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:40 smithi018 ceph-mon[23740]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:40 smithi018 ceph-mon[28324]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:40 smithi031 ceph-mon[24497]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:42 smithi031 ceph-mon[24497]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:43.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:42 smithi018 ceph-mon[23740]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:42 smithi018 ceph-mon[28324]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:56:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:56:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:44 smithi031 ceph-mon[24497]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:45.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:44 smithi018 ceph-mon[28324]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:44 smithi018 ceph-mon[23740]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:46 smithi018 ceph-mon[23740]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:46 smithi018 ceph-mon[28324]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:46 smithi031 ceph-mon[24497]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:48.965 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[23740]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:48.965 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:56:48.966 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:56:48.966 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[28324]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:48.966 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:56:48.966 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:48 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:56:48.970 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:48 smithi031 ceph-mon[24497]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:48.970 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:48 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:56:48.971 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:48 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:56:49.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:49] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-20T23:56:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:56:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:56:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:56:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:50 smithi031 ceph-mon[24497]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:50 smithi018 ceph-mon[23740]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:50 smithi018 ceph-mon[28324]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:52 smithi031 ceph-mon[24497]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:53.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:56:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:56:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:52 smithi018 ceph-mon[23740]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:52 smithi018 ceph-mon[28324]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:54 smithi031 ceph-mon[24497]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:55.403 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:54 smithi018 ceph-mon[23740]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:55.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:54 smithi018 ceph-mon[28324]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:56 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:56 smithi018 ceph-mon[23740]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:56 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:56 smithi018 ceph-mon[28324]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:56 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:56:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:56 smithi031 ceph-mon[24497]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:56:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:56:58 smithi018 ceph-mon[23740]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:56:58 smithi018 ceph-mon[28324]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:56:59.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:56:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:56:59] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-20T23:56:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:56:58 smithi031 ceph-mon[24497]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:01.057 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:00 smithi031 ceph-mon[24497]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:00 smithi018 ceph-mon[23740]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:00 smithi018 ceph-mon[28324]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:03.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:02 smithi031 ceph-mon[24497]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:02 smithi018 ceph-mon[23740]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:02 smithi018 ceph-mon[28324]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:03.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:04 smithi018 ceph-mon[23740]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:04 smithi018 ceph-mon[28324]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:04 smithi031 ceph-mon[24497]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:06 smithi018 ceph-mon[23740]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:06 smithi018 ceph-mon[28324]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:06 smithi031 ceph-mon[24497]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:08 smithi018 ceph-mon[23740]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:08 smithi018 ceph-mon[28324]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:09.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:09] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-20T23:57:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:08 smithi031 ceph-mon[24497]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:10 smithi018 ceph-mon[23740]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:10 smithi018 ceph-mon[28324]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:10 smithi031 ceph-mon[24497]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:13.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:12 smithi031 ceph-mon[24497]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:12 smithi018 ceph-mon[23740]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:12 smithi018 ceph-mon[28324]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:13.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:14 smithi018 ceph-mon[23740]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:14 smithi018 ceph-mon[28324]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:14 smithi031 ceph-mon[24497]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:16 smithi018 ceph-mon[23740]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:16 smithi018 ceph-mon[28324]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:16 smithi031 ceph-mon[24497]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:18 smithi018 ceph-mon[23740]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:18 smithi018 ceph-mon[28324]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:19.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:19] "GET /metrics HTTP/1.1" 200 207990 "" "Prometheus/2.33.4" 2024-09-20T23:57:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:18 smithi031 ceph-mon[24497]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:20 smithi018 ceph-mon[23740]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:20 smithi018 ceph-mon[28324]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:20 smithi031 ceph-mon[24497]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:23.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:22 smithi031 ceph-mon[24497]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:22 smithi018 ceph-mon[23740]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:22 smithi018 ceph-mon[28324]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:23.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:24 smithi018 ceph-mon[23740]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:24 smithi018 ceph-mon[28324]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:24 smithi031 ceph-mon[24497]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:26 smithi018 ceph-mon[23740]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:26 smithi018 ceph-mon[28324]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:26 smithi031 ceph-mon[24497]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:28 smithi018 ceph-mon[23740]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:28 smithi018 ceph-mon[28324]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:29.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:29] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-09-20T23:57:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:28 smithi031 ceph-mon[24497]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:30 smithi018 ceph-mon[23740]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:30 smithi018 ceph-mon[28324]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:30 smithi031 ceph-mon[24497]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:33.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:32 smithi031 ceph-mon[24497]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:32 smithi018 ceph-mon[23740]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:32 smithi018 ceph-mon[28324]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:33.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:34 smithi018 ceph-mon[23740]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:34 smithi018 ceph-mon[28324]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:34 smithi031 ceph-mon[24497]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:36 smithi018 ceph-mon[23740]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:36 smithi018 ceph-mon[28324]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:36 smithi031 ceph-mon[24497]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:38 smithi018 ceph-mon[23740]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:38 smithi018 ceph-mon[28324]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:39.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:39] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-09-20T23:57:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:38 smithi031 ceph-mon[24497]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:40 smithi018 ceph-mon[23740]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:40 smithi018 ceph-mon[28324]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:40 smithi031 ceph-mon[24497]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:43.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:42 smithi031 ceph-mon[24497]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:42 smithi018 ceph-mon[23740]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:42 smithi018 ceph-mon[28324]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:43.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:57:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:57:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:44 smithi018 ceph-mon[23740]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:44 smithi018 ceph-mon[28324]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:44 smithi031 ceph-mon[24497]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:46 smithi018 ceph-mon[23740]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:46 smithi018 ceph-mon[28324]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:46 smithi031 ceph-mon[24497]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:48 smithi018 ceph-mon[23740]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:49.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:49] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-09-20T23:57:49.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:48 smithi018 ceph-mon[28324]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:48 smithi031 ceph-mon[24497]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:57:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:57:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:57:50.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:57:50.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:57:50.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:57:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:57:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:57:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:57:51.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:51 smithi018 ceph-mon[23740]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:51.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:51 smithi018 ceph-mon[28324]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:51.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:51 smithi031 ceph-mon[24497]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:53 smithi031 ceph-mon[24497]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:53.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:57:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:57:53.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:53 smithi018 ceph-mon[23740]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:53.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:53 smithi018 ceph-mon[28324]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:55 smithi018 ceph-mon[23740]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:55 smithi018 ceph-mon[28324]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:55 smithi031 ceph-mon[24497]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:56 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:56 smithi018 ceph-mon[23740]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:56 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:56 smithi018 ceph-mon[28324]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:56 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:57:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:56 smithi031 ceph-mon[24497]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:57:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:57:58 smithi018 ceph-mon[23740]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:57:58 smithi018 ceph-mon[28324]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:57:59.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:57:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:57:59] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-09-20T23:57:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:57:58 smithi031 ceph-mon[24497]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:00 smithi018 ceph-mon[23740]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:00 smithi018 ceph-mon[28324]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:00 smithi031 ceph-mon[24497]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:03.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:02 smithi031 ceph-mon[24497]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:02 smithi018 ceph-mon[23740]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:02 smithi018 ceph-mon[28324]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:03.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:04 smithi018 ceph-mon[23740]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:04 smithi018 ceph-mon[28324]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:04 smithi031 ceph-mon[24497]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:06 smithi018 ceph-mon[28324]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:06 smithi018 ceph-mon[23740]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:06 smithi031 ceph-mon[24497]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:08 smithi018 ceph-mon[23740]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:08 smithi018 ceph-mon[28324]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:09.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:09] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-09-20T23:58:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:08 smithi031 ceph-mon[24497]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:10 smithi018 ceph-mon[23740]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:10 smithi018 ceph-mon[28324]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:11.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:10 smithi031 ceph-mon[24497]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:13.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:12 smithi031 ceph-mon[24497]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:12 smithi018 ceph-mon[23740]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:12 smithi018 ceph-mon[28324]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:13.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:14 smithi018 ceph-mon[23740]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:14 smithi018 ceph-mon[28324]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:14 smithi031 ceph-mon[24497]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:16 smithi018 ceph-mon[23740]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:16 smithi018 ceph-mon[28324]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:16 smithi031 ceph-mon[24497]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:18 smithi018 ceph-mon[23740]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:18 smithi018 ceph-mon[28324]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:19.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:19] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-09-20T23:58:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:18 smithi031 ceph-mon[24497]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:20 smithi018 ceph-mon[23740]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:20 smithi018 ceph-mon[28324]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:20 smithi031 ceph-mon[24497]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:23.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:22 smithi031 ceph-mon[24497]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:22 smithi018 ceph-mon[23740]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:22 smithi018 ceph-mon[28324]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:23.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:24 smithi018 ceph-mon[23740]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:24 smithi018 ceph-mon[28324]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:24 smithi031 ceph-mon[24497]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:26 smithi018 ceph-mon[23740]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:26.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:26 smithi018 ceph-mon[28324]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:26 smithi031 ceph-mon[24497]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:28 smithi018 ceph-mon[23740]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:28 smithi018 ceph-mon[28324]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:29.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:29] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-09-20T23:58:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:28 smithi031 ceph-mon[24497]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:30 smithi018 ceph-mon[23740]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:30 smithi018 ceph-mon[28324]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:30 smithi031 ceph-mon[24497]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:33.111 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:32 smithi031 ceph-mon[24497]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:32 smithi018 ceph-mon[23740]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:32 smithi018 ceph-mon[28324]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:33.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:34 smithi018 ceph-mon[23740]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:34 smithi018 ceph-mon[28324]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:34 smithi031 ceph-mon[24497]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:36 smithi018 ceph-mon[23740]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:36 smithi018 ceph-mon[28324]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:36 smithi031 ceph-mon[24497]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:38 smithi018 ceph-mon[23740]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:38 smithi018 ceph-mon[28324]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:39.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:39] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-09-20T23:58:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:38 smithi031 ceph-mon[24497]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:40 smithi018 ceph-mon[23740]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:40 smithi018 ceph-mon[28324]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:40 smithi031 ceph-mon[24497]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:43.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:42 smithi031 ceph-mon[24497]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:42 smithi018 ceph-mon[23740]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:42 smithi018 ceph-mon[28324]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:43.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:58:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:58:44.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-20T23:58:45.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:44 smithi018 ceph-mon[23740]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:45.157 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:44 smithi018 ceph-mon[28324]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:44 smithi031 ceph-mon[24497]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:46.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:46 smithi018 ceph-mon[23740]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:46.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:46.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:46 smithi018 ceph-mon[28324]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:46 smithi031 ceph-mon[24497]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:47.494 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-20T23:58:49.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:48 smithi018 ceph-mon[23740]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:49.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:48 smithi018 ceph-mon[28324]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:48 smithi031 ceph-mon[24497]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:49.297 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:49] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-09-20T23:58:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:58:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:58:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:58:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:58:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:58:49.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:49 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:58:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:58:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:49 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:58:50.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:49 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:58:50.241 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-20T23:58:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:50 smithi018 ceph-mon[23740]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:50 smithi018 ceph-mon[28324]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:50 smithi031 ceph-mon[24497]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:52.662 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-20T23:58:52.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:52 smithi018 ceph-mon[23740]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:52.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:52 smithi018 ceph-mon[28324]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:53.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:52 smithi031 ceph-mon[24497]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:53.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:58:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:58:54.483 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T23:58:54.483 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (3m) 3m ago 4m 15.8M - ba2b418f427c 11c7d61a9d98 2024-09-20T23:58:54.483 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (3m) 3m ago 3m 47.5M - 8.3.5 dad864ee21e9 9103855ea14b 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (3m) 3m ago 3m 60.3M - 3.5 e1d6a67b021e 1373637a0def 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e df2070968285 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 768baae6a4fb 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (10m) 3m ago 10m 45.4M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (8m) 3m ago 8m 38.9M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (9m) 3m ago 9m 39.9M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (4m) 3m ago 4m 17.5M - 1dbe0e931976 b8ef2efc32f5 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (4m) 3m ago 4m 17.8M - 1dbe0e931976 95837fdf8e6d 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (8m) 3m ago 8m 47.3M 4017M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (8m) 3m ago 8m 47.2M 4017M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (7m) 3m ago 7m 46.6M 4017M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-20T23:58:54.484 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (7m) 3m ago 7m 44.7M 4017M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (6m) 3m ago 6m 46.5M 4273M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (6m) 3m ago 6m 43.4M 4273M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (6m) 3m ago 6m 43.3M 4273M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (5m) 3m ago 5m 45.2M 4273M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (3m) 3m ago 4m 42.2M - 514e6a882f6e ccdbacac91c5 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-20T23:58:54.485 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-20T23:58:55.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:54 smithi018 ceph-mon[28324]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:55.118 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:54 smithi018 ceph-mon[23740]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:55.143 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-20T23:58:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:54 smithi031 ceph-mon[24497]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:56.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:55 smithi018 ceph-mon[28324]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:58:56.109 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:55 smithi018 ceph-mon[23740]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:58:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:55 smithi031 ceph-mon[24497]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:58:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:56 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:56 smithi018 ceph-mon[23740]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:56 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:56 smithi018 ceph-mon[28324]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-20T23:58:57.211 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T23:58:57.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-20T23:58:57.212 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-20T23:58:57.212 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-09-20T23:58:57.212 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-20T23:58:57.212 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-20T23:58:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:56 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:58:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:56 smithi031 ceph-mon[24497]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:58:57.852 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph -s' 2024-09-20T23:58:57.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:57 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1300027278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T23:58:57.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:57 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1300027278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T23:58:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:57 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1300027278' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T23:58:59.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:58 smithi018 ceph-mon[23740]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:59.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:58 smithi018 ceph-mon[28324]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:58 smithi031 ceph-mon[24497]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:58:59.367 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:58:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:58:59] "GET /metrics HTTP/1.1" 200 207935 "" "Prometheus/2.33.4" 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: cluster: 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: id: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: health: HEALTH_OK 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: services: 2024-09-20T23:58:59.830 INFO:teuthology.orchestra.run.smithi018.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: mgr: y(active, since 5m), standbys: x 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: data: 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: pools: 6 pools, 161 pgs 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: objects: 209 objects, 585 KiB 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: pgs: 161 active+clean 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: io: 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-20T23:58:59.831 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-20T23:59:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:58:59 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1864908689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T23:59:00.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:58:59 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1864908689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T23:59:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:58:59 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1864908689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T23:59:00.497 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ls' 2024-09-20T23:59:01.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:00 smithi018 ceph-mon[23740]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:01.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:00 smithi018 ceph-mon[28324]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:01.152 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:00 smithi031 ceph-mon[24497]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi018=a;count:1 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:grafana ?:3000 1/1 3m ago 5m smithi031=a;count:1 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:mgr 2/2 3m ago 8m smithi018=y;smithi031=x;count:2 2024-09-20T23:59:02.392 INFO:teuthology.orchestra.run.smithi018.stdout:mon 3/3 3m ago 9m smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi031:172.21.15.31=b;count:3 2024-09-20T23:59:02.393 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi018=a;smithi031=b;count:2 2024-09-20T23:59:02.393 INFO:teuthology.orchestra.run.smithi018.stdout:osd 8 3m ago - 2024-09-20T23:59:02.393 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus ?:9095 1/1 3m ago 5m smithi031=a;count:1 2024-09-20T23:59:02.393 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-09-20T23:59:02.987 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-20T23:59:03.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:02 smithi031 ceph-mon[24497]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:03.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:02 smithi031 ceph-mon[24497]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:02 smithi018 ceph-mon[23740]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:02 smithi018 ceph-mon[23740]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:02 smithi018 ceph-mon[28324]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:02 smithi018 ceph-mon[28324]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:03.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:59:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:04 smithi018 ceph-mon[23740]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:04 smithi018 ceph-mon[28324]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:04 smithi031 ceph-mon[24497]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:05.421 INFO:teuthology.orchestra.run.smithi018.stdout:Scheduled to redeploy mgr.x on host 'smithi031' 2024-09-20T23:59:06.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/92401948' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:59:06.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:06.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:06.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/92401948' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:59:06.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:05 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:06.128 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps --refresh' 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/92401948' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T23:59:06.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T23:59:06.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:05 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[28324]: Schedule redeploy daemon mgr.x 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[28324]: Deploying daemon mgr.x on smithi031 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[28324]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[23740]: Schedule redeploy daemon mgr.x 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[23740]: Deploying daemon mgr.x on smithi031 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:07.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:06 smithi018 ceph-mon[23740]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:06 smithi031 ceph-mon[24497]: Schedule redeploy daemon mgr.x 2024-09-20T23:59:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:06 smithi031 ceph-mon[24497]: Deploying daemon mgr.x on smithi031 2024-09-20T23:59:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:06 smithi031 ceph-mon[24497]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:08.201 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T23:59:08.201 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (3m) 3m ago 4m 15.8M - ba2b418f427c 11c7d61a9d98 2024-09-20T23:59:08.201 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (4m) 3m ago 4m 47.5M - 8.3.5 dad864ee21e9 9103855ea14b 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (3m) 3m ago 3m 60.3M - 3.5 e1d6a67b021e 1373637a0def 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e df2070968285 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 768baae6a4fb 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (10m) 3m ago 10m 45.4M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (9m) 3m ago 9m 38.9M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (9m) 3m ago 9m 39.9M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (5m) 3m ago 5m 17.5M - 1dbe0e931976 b8ef2efc32f5 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (5m) 3m ago 5m 17.8M - 1dbe0e931976 95837fdf8e6d 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (8m) 3m ago 8m 47.3M 4017M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (8m) 3m ago 8m 47.2M 4017M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (7m) 3m ago 7m 46.6M 4017M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (7m) 3m ago 7m 44.7M 4017M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-20T23:59:08.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (7m) 3m ago 7m 46.5M 4273M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (6m) 3m ago 6m 43.4M 4273M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (6m) 3m ago 6m 43.3M 4273M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (5m) 3m ago 5m 45.2M 4273M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (3m) 3m ago 5m 42.2M - 514e6a882f6e ccdbacac91c5 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-20T23:59:08.203 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-20T23:59:08.861 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-20T23:59:09.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:08 smithi018 ceph-mon[23740]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:09.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:08 smithi018 ceph-mon[23740]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:09.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:08 smithi018 ceph-mon[28324]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:09.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:08 smithi018 ceph-mon[28324]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:08 smithi031 ceph-mon[24497]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:08 smithi031 ceph-mon[24497]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T23:59:09.321 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:09] "GET /metrics HTTP/1.1" 200 207935 "" "Prometheus/2.33.4" 2024-09-20T23:59:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:10 smithi018 ceph-mon[23740]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:11.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:10 smithi018 ceph-mon[28324]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:10 smithi031 ceph-mon[24497]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:12 smithi018 ceph-mon[23740]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:12 smithi018 ceph-mon[28324]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:12 smithi031 ceph-mon[24497]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:13.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:59:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:14 smithi018 ceph-mon[23740]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:14 smithi018 ceph-mon[28324]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:14 smithi031 ceph-mon[24497]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:16 smithi018 ceph-mon[23740]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:16 smithi018 ceph-mon[28324]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:16 smithi031 ceph-mon[24497]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:18 smithi018 ceph-mon[23740]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:18 smithi018 ceph-mon[28324]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:19.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:19] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-09-20T23:59:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:18 smithi031 ceph-mon[24497]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:20 smithi018 ceph-mon[23740]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:20 smithi018 ceph-mon[28324]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:20 smithi031 ceph-mon[24497]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:23.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:22 smithi031 ceph-mon[24497]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:22 smithi018 ceph-mon[23740]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:22 smithi018 ceph-mon[28324]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:23.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:59:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:24 smithi018 ceph-mon[23740]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:24 smithi018 ceph-mon[28324]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:24 smithi031 ceph-mon[24497]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:26 smithi018 ceph-mon[23740]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:26 smithi018 ceph-mon[28324]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:26 smithi031 ceph-mon[24497]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:28 smithi018 ceph-mon[23740]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:28 smithi018 ceph-mon[28324]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:29.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:29] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-09-20T23:59:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:28 smithi031 ceph-mon[24497]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:30 smithi018 ceph-mon[23740]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:30 smithi018 ceph-mon[28324]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:30 smithi031 ceph-mon[24497]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:33.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:32 smithi031 ceph-mon[24497]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:32 smithi018 ceph-mon[23740]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:32 smithi018 ceph-mon[28324]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:33.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:59:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:34 smithi018 ceph-mon[23740]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:34 smithi018 ceph-mon[28324]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:34 smithi031 ceph-mon[24497]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:36 smithi018 ceph-mon[23740]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:36 smithi018 ceph-mon[28324]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:36 smithi031 ceph-mon[24497]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:38 smithi018 ceph-mon[23740]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:39.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:39] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-09-20T23:59:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:38 smithi018 ceph-mon[28324]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:38 smithi031 ceph-mon[24497]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:40 smithi018 ceph-mon[23740]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:40 smithi018 ceph-mon[28324]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:40 smithi031 ceph-mon[24497]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:43.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:42 smithi031 ceph-mon[24497]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:42 smithi018 ceph-mon[23740]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:42 smithi018 ceph-mon[28324]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:43.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[25662]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-20T23:59:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-20T23:59:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-20T23:59:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:44 smithi018 ceph-mon[23740]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:44 smithi018 ceph-mon[28324]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:44 smithi031 ceph-mon[24497]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:46 smithi018 ceph-mon[23740]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:46 smithi018 ceph-mon[28324]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:46 smithi031 ceph-mon[24497]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:47.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:47 smithi031 systemd[1]: Stopping Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:59:47.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:47 smithi031 bash[44043]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr.x" found: no such container 2024-09-20T23:59:47.860 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:47 smithi031 podman[44054]: 2024-09-20 23:59:47.553845156 +0000 UTC m=+0.111496687 container died df207096828523bfdde07549d12cfdcccde82c9e03ebe1eaf581a2f834440de7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, version=8) 2024-09-20T23:59:48.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44054]: 2024-09-20 23:59:48.009112962 +0000 UTC m=+0.566764481 container cleanup df207096828523bfdde07549d12cfdcccde82c9e03ebe1eaf581a2f834440de7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, com.redhat.component=centos-stream-container, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-type=git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , ceph=True) 2024-09-20T23:59:48.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 bash[44054]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x 2024-09-20T23:59:48.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 bash[44080]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr.x" found: no such container 2024-09-20T23:59:48.688 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44066]: 2024-09-20 23:59:48.260259725 +0000 UTC m=+0.700434141 container remove df207096828523bfdde07549d12cfdcccde82c9e03ebe1eaf581a2f834440de7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T23:59:48.688 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-20T23:59:48.688 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-20T23:59:48.688 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 systemd[1]: Stopped Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:59:48.689 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Consumed 17.955s CPU time. 2024-09-20T23:59:48.689 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 systemd[1]: Starting Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-20T23:59:48.965 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:48 smithi031 ceph-mon[24497]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:48.965 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44180]: 2024-09-20 23:59:48.590466533 +0000 UTC m=+0.016695963 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-20T23:59:48.965 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44180]: 2024-09-20 23:59:48.722186959 +0000 UTC m=+0.148416400 container create 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T23:59:48.965 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44180]: 2024-09-20 23:59:48.964498897 +0000 UTC m=+0.390728318 container init 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-09-20T23:59:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:48 smithi018 ceph-mon[23740]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:48 smithi018 ceph-mon[28324]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:49.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:49] "GET /metrics HTTP/1.1" 200 207955 "" "Prometheus/2.33.4" 2024-09-20T23:59:49.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 podman[44180]: 2024-09-20 23:59:48.967975638 +0000 UTC m=+0.394205037 container start 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70) 2024-09-20T23:59:49.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:48 smithi031 ceph-mgr[44199]: -- 172.21.15.31:0/2994922267 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55af192cf0e0 con 0x55af192ac800 2024-09-20T23:59:49.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 bash[44180]: 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a 2024-09-20T23:59:49.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 systemd[1]: Started Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-20T23:59:49.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.092+0000 7f3f3290c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-20T23:59:49.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.159+0000 7f3f3290c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-20T23:59:49.722 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.322+0000 7f3f3290c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-20T23:59:49.722 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.398+0000 7f3f3290c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-20T23:59:49.722 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.561+0000 7f3f3290c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:59:49.722 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:49.720+0000 7f3f3290c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:50.069+0000 7f3f3290c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:50.250+0000 7f3f3290c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T23:59:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T23:59:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:50 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:50.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:50.447+0000 7f3f3290c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-20T23:59:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:51 smithi018 ceph-mon[23740]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:51 smithi018 ceph-mon[28324]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:51.452 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:51 smithi031 ceph-mon[24497]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:51.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:51.450+0000 7f3f3290c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-20T23:59:51.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:51.530+0000 7f3f3290c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-20T23:59:52.613 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.325+0000 7f3f3290c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-20T23:59:52.613 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.460+0000 7f3f3290c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-20T23:59:52.613 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.611+0000 7f3f3290c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-20T23:59:52.879 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.679+0000 7f3f3290c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-20T23:59:52.879 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.837+0000 7f3f3290c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-20T23:59:53.146 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.909+0000 7f3f3290c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-20T23:59:53.147 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:52.972+0000 7f3f3290c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-20T23:59:53.147 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.145+0000 7f3f3290c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-20T23:59:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:53 smithi018 ceph-mon[23740]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:53 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:53 smithi018 ceph-mon[28324]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:53 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:53.448 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-mon[24497]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:53.448 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:53.449 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.218+0000 7f3f3290c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-20T23:59:53.449 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.288+0000 7f3f3290c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-20T23:59:53.449 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.383+0000 7f3f3290c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-20T23:59:53.449 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.447+0000 7f3f3290c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-20T23:59:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:53.509+0000 7f3f3290c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-20T23:59:54.474 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:54.365+0000 7f3f3290c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-20T23:59:54.474 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [20/Sep/2024:23:59:54] ENGINE Bus STARTING 2024-09-20T23:59:54.474 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: CherryPy Checker: 2024-09-20T23:59:54.474 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: The Application mounted at '' has an empty config. 2024-09-20T23:59:54.475 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-20T23:59:54.475 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [20/Sep/2024:23:59:54] ENGINE Serving on http://:::9283 2024-09-20T23:59:54.475 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [20/Sep/2024:23:59:54] ENGINE Bus STARTED 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:59:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: Standby manager daemon x restarted 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:59:55.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:54 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: Standby manager daemon x restarted 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-20T23:59:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-20T23:59:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T23:59:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-20T23:59:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:54 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T23:59:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:56 smithi031 ceph-mon[24497]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-20T23:59:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:56 smithi018 ceph-mon[23740]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-20T23:59:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:56 smithi018 ceph-mon[28324]: mgrmap e21: y(active, since 6m), standbys: x 2024-09-20T23:59:57.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:57 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:57.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:57 smithi031 ceph-mon[24497]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:57 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:57 smithi018 ceph-mon[23740]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:57 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-20T23:59:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:57 smithi018 ceph-mon[28324]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-20T23:59:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 20 23:59:59 smithi018 ceph-mon[23740]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 20 23:59:59 smithi018 ceph-mon[28324]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-20T23:59:59.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 20 23:59:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [20/Sep/2024:23:59:59] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-09-20T23:59:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 20 23:59:59 smithi031 ceph-mon[24497]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:00 smithi031 ceph-mon[24497]: overall HEALTH_OK 2024-09-21T00:00:00.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:00 smithi018 ceph-mon[23740]: overall HEALTH_OK 2024-09-21T00:00:00.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:00 smithi018 ceph-mon[28324]: overall HEALTH_OK 2024-09-21T00:00:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:01 smithi031 ceph-mon[24497]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:01 smithi018 ceph-mon[23740]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:01.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:01 smithi018 ceph-mon[28324]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:03 smithi018 ceph-mon[23740]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:03 smithi018 ceph-mon[28324]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:03.470 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:03 smithi031 ceph-mon[24497]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:03.470 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:05 smithi018 ceph-mon[23740]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:05 smithi018 ceph-mon[28324]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:05 smithi031 ceph-mon[24497]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:06 smithi018 ceph-mon[23740]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 50 op/s 2024-09-21T00:00:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:06 smithi018 ceph-mon[28324]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 50 op/s 2024-09-21T00:00:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:06 smithi031 ceph-mon[24497]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 50 op/s 2024-09-21T00:00:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:08 smithi018 ceph-mon[23740]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 49 op/s 2024-09-21T00:00:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:08 smithi018 ceph-mon[28324]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 49 op/s 2024-09-21T00:00:09.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:09] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-09-21T00:00:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:08 smithi031 ceph-mon[24497]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 72 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 49 op/s 2024-09-21T00:00:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:10 smithi018 ceph-mon[28324]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:10 smithi018 ceph-mon[23740]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:10 smithi031 ceph-mon[24497]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:12 smithi018 ceph-mon[23740]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:12 smithi018 ceph-mon[28324]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:12 smithi031 ceph-mon[24497]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:13.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:00:13.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:14 smithi018 ceph-mon[23740]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:14 smithi018 ceph-mon[28324]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:14 smithi031 ceph-mon[24497]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-21T00:00:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:16 smithi018 ceph-mon[23740]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 97 op/s 2024-09-21T00:00:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:16 smithi018 ceph-mon[28324]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 97 op/s 2024-09-21T00:00:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:16 smithi031 ceph-mon[24497]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 97 op/s 2024-09-21T00:00:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:18 smithi018 ceph-mon[23740]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 47 op/s 2024-09-21T00:00:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:18 smithi018 ceph-mon[28324]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 47 op/s 2024-09-21T00:00:19.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:19] "GET /metrics HTTP/1.1" 200 208058 "" "Prometheus/2.33.4" 2024-09-21T00:00:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:18 smithi031 ceph-mon[24497]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 47 op/s 2024-09-21T00:00:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:20 smithi018 ceph-mon[23740]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 48 op/s 2024-09-21T00:00:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:20 smithi018 ceph-mon[28324]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 48 op/s 2024-09-21T00:00:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:20 smithi031 ceph-mon[24497]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 48 op/s 2024-09-21T00:00:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:22 smithi018 ceph-mon[23740]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:22 smithi018 ceph-mon[28324]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:22 smithi031 ceph-mon[24497]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:23.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:23.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:00:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:23.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:23.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:23.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:25.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:24 smithi018 ceph-mon[23740]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:25.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:24 smithi018 ceph-mon[28324]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:24 smithi031 ceph-mon[24497]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:26 smithi018 ceph-mon[23740]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:26 smithi018 ceph-mon[28324]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:26 smithi031 ceph-mon[24497]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T00:00:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:28 smithi018 ceph-mon[28324]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:28 smithi018 ceph-mon[23740]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:29.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:29] "GET /metrics HTTP/1.1" 200 208048 "" "Prometheus/2.33.4" 2024-09-21T00:00:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:28 smithi031 ceph-mon[24497]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:30 smithi018 ceph-mon[23740]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:30 smithi018 ceph-mon[28324]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:30 smithi031 ceph-mon[24497]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:32 smithi018 ceph-mon[23740]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:32 smithi018 ceph-mon[28324]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:32 smithi031 ceph-mon[24497]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:33.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:00:33.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:33.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:34 smithi018 ceph-mon[23740]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:34 smithi018 ceph-mon[28324]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:34 smithi031 ceph-mon[24497]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:36 smithi018 ceph-mon[23740]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:36 smithi018 ceph-mon[28324]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:36 smithi031 ceph-mon[24497]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:38 smithi018 ceph-mon[23740]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:38 smithi018 ceph-mon[28324]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:39.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:39] "GET /metrics HTTP/1.1" 200 208048 "" "Prometheus/2.33.4" 2024-09-21T00:00:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:38 smithi031 ceph-mon[24497]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:40 smithi018 ceph-mon[23740]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:40 smithi018 ceph-mon[28324]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:40 smithi031 ceph-mon[24497]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:42 smithi018 ceph-mon[28324]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:42 smithi018 ceph-mon[23740]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:42 smithi031 ceph-mon[24497]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:43.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:43.883 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:00:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:43.883 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:43.883 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:44.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:00:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:00:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:44 smithi018 ceph-mon[23740]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:44 smithi018 ceph-mon[28324]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:44 smithi031 ceph-mon[24497]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:46 smithi018 ceph-mon[28324]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:46 smithi018 ceph-mon[23740]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:46 smithi031 ceph-mon[24497]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:48 smithi018 ceph-mon[23740]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:49.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:48 smithi018 ceph-mon[28324]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:49.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:49] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-09-21T00:00:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:48 smithi031 ceph-mon[24497]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:50 smithi018 ceph-mon[23740]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:50 smithi018 ceph-mon[28324]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:50 smithi031 ceph-mon[24497]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:52 smithi018 ceph-mon[23740]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:52 smithi018 ceph-mon[28324]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:52 smithi031 ceph-mon[24497]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:53.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:00:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:00:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:00:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:00:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:00:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:00:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:00:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:54 smithi031 ceph-mon[24497]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:54 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:00:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:54 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:00:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[23740]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:00:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:00:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[28324]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:00:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:54 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:00:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:56 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:56 smithi031 ceph-mon[24497]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:56 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:56 smithi018 ceph-mon[23740]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:56 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:00:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:56 smithi018 ceph-mon[28324]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:00:59.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:00:59] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-09-21T00:00:59.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:59 smithi031 ceph-mon[24497]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:59.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:59 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:00:59.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:00:59 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:00:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[23740]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:00:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:00:59.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[28324]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:00:59.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:00:59.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:00:59 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:01:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:00 smithi031 ceph-mon[24497]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:00.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:00 smithi018 ceph-mon[23740]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:00 smithi018 ceph-mon[28324]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:02 smithi018 ceph-mon[23740]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:02 smithi018 ceph-mon[28324]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:02 smithi031 ceph-mon[24497]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:03.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:04 smithi018 ceph-mon[23740]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:04 smithi018 ceph-mon[28324]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:04 smithi031 ceph-mon[24497]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:06 smithi018 ceph-mon[23740]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:06 smithi018 ceph-mon[28324]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:06 smithi031 ceph-mon[24497]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:08 smithi018 ceph-mon[23740]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:08 smithi018 ceph-mon[28324]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:09.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:09] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-09-21T00:01:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:08 smithi031 ceph-mon[24497]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:10 smithi018 ceph-mon[28324]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:10 smithi018 ceph-mon[23740]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:10 smithi031 ceph-mon[24497]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:12 smithi018 ceph-mon[23740]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:12 smithi018 ceph-mon[28324]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:12 smithi031 ceph-mon[24497]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:13.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:13.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:14 smithi018 ceph-mon[23740]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:14 smithi018 ceph-mon[28324]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:14 smithi031 ceph-mon[24497]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:16 smithi018 ceph-mon[23740]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:16 smithi018 ceph-mon[28324]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:16 smithi031 ceph-mon[24497]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:18 smithi018 ceph-mon[23740]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:18 smithi018 ceph-mon[28324]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:19.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:19] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-09-21T00:01:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:18 smithi031 ceph-mon[24497]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:20 smithi018 ceph-mon[23740]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:20 smithi018 ceph-mon[28324]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:20 smithi031 ceph-mon[24497]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:22 smithi018 ceph-mon[23740]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:22 smithi018 ceph-mon[28324]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:22 smithi031 ceph-mon[24497]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:23.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:23.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:23.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:23.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:23.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:24 smithi018 ceph-mon[23740]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:24 smithi018 ceph-mon[28324]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:24 smithi031 ceph-mon[24497]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:26 smithi018 ceph-mon[23740]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:26 smithi018 ceph-mon[28324]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:26 smithi031 ceph-mon[24497]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:28 smithi018 ceph-mon[23740]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:28 smithi018 ceph-mon[28324]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:29.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:29] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-09-21T00:01:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:28 smithi031 ceph-mon[24497]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:30 smithi018 ceph-mon[23740]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:30 smithi018 ceph-mon[28324]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:30 smithi031 ceph-mon[24497]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:32 smithi018 ceph-mon[23740]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:32 smithi018 ceph-mon[28324]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:32 smithi031 ceph-mon[24497]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:33.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:33.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:33.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:34 smithi018 ceph-mon[23740]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:34 smithi018 ceph-mon[28324]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:34 smithi031 ceph-mon[24497]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:36 smithi018 ceph-mon[23740]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:36.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:36 smithi018 ceph-mon[28324]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:37.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:37.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:36 smithi031 ceph-mon[24497]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:38 smithi018 ceph-mon[23740]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:38 smithi018 ceph-mon[28324]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:39.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:39] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-09-21T00:01:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:38 smithi031 ceph-mon[24497]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:40 smithi018 ceph-mon[23740]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:40 smithi018 ceph-mon[28324]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:40 smithi031 ceph-mon[24497]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:42 smithi018 ceph-mon[23740]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:42 smithi018 ceph-mon[28324]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:42 smithi031 ceph-mon[24497]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:43.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:43.896 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:43.897 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:43.897 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:43 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:43 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:01:44.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:43 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:01:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:44 smithi018 ceph-mon[23740]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:44 smithi018 ceph-mon[28324]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:44 smithi031 ceph-mon[24497]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:46.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:46 smithi018 ceph-mon[23740]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:46.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:46 smithi018 ceph-mon[28324]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:46 smithi031 ceph-mon[24497]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:48 smithi018 ceph-mon[23740]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:48 smithi018 ceph-mon[28324]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:49.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:49] "GET /metrics HTTP/1.1" 200 208030 "" "Prometheus/2.33.4" 2024-09-21T00:01:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:48 smithi031 ceph-mon[24497]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:50 smithi018 ceph-mon[23740]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:50 smithi018 ceph-mon[28324]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:50 smithi031 ceph-mon[24497]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:52 smithi018 ceph-mon[23740]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:52 smithi018 ceph-mon[28324]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:52 smithi031 ceph-mon[24497]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:53.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:01:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:01:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:01:53.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:01:53.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:01:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:01:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:01:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:54 smithi018 ceph-mon[23740]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:54 smithi018 ceph-mon[28324]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:54 smithi031 ceph-mon[24497]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:56 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:56 smithi018 ceph-mon[23740]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:56 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:56.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:56 smithi018 ceph-mon[28324]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:56 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:01:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:56 smithi031 ceph-mon[24497]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:01:59.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[23740]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:59.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:01:59.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[23740]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:01:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[28324]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:01:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:01:58 smithi018 ceph-mon[28324]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:01:59.079 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:58 smithi031 ceph-mon[24497]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:01:59.079 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:58 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:01:59.079 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:01:58 smithi031 ceph-mon[24497]: from='mgr.24353 172.21.15.18:0/775825929' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:01:59.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:01:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:01:59] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-09-21T00:02:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:00 smithi018 ceph-mon[23740]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:02:00.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:00 smithi018 ceph-mon[28324]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:02:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:00 smithi031 ceph-mon[24497]: from='mgr.24353 ' entity='mgr.y' 2024-09-21T00:02:01.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:01 smithi031 ceph-mon[24497]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:01 smithi018 ceph-mon[23740]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:01.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:01 smithi018 ceph-mon[28324]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:03.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:03 smithi018 ceph-mon[23740]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:03.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:03 smithi018 ceph-mon[28324]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:03.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:03 smithi031 ceph-mon[24497]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:03.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:02:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:02:03.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:03.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:05 smithi018 ceph-mon[23740]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:05 smithi018 ceph-mon[28324]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:05 smithi031 ceph-mon[24497]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:06 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:06 smithi018 ceph-mon[23740]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:06 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:06 smithi018 ceph-mon[28324]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:06 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:06 smithi031 ceph-mon[24497]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:08 smithi018 ceph-mon[23740]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:08 smithi018 ceph-mon[28324]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:09.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:09 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:09] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-09-21T00:02:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:08 smithi031 ceph-mon[24497]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:10.977 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:10 smithi018 ceph-mon[23740]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:10.978 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:10 smithi018 ceph-mon[28324]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:10 smithi031 ceph-mon[24497]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:11.548 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:02:13.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:12 smithi018 ceph-mon[23740]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:13.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:12 smithi018 ceph-mon[28324]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:12 smithi031 ceph-mon[24497]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:13.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:02:13.652 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (6m) 2m ago 7m 25.8M - ba2b418f427c 11c7d61a9d98 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (7m) 75s ago 7m 56.6M - 8.3.5 dad864ee21e9 9103855ea14b 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (6m) 2m ago 6m 47.0M - 3.5 e1d6a67b021e 1373637a0def 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (2m) 75s ago 12m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:9283 running (13m) 2m ago 13m 460M - 17.2.0 e1d6a67b021e 768baae6a4fb 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (13m) 2m ago 13m 49.3M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (12m) 75s ago 12m 39.5M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (12m) 2m ago 12m 42.0M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (8m) 2m ago 8m 17.1M - 1dbe0e931976 b8ef2efc32f5 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (8m) 75s ago 8m 17.2M - 1dbe0e931976 95837fdf8e6d 2024-09-21T00:02:13.653 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (11m) 2m ago 11m 49.9M 4017M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (11m) 2m ago 11m 49.7M 4017M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (10m) 2m ago 10m 47.8M 4017M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (10m) 2m ago 10m 47.8M 4017M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (10m) 75s ago 10m 50.9M 4273M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (9m) 75s ago 9m 48.7M 4273M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (9m) 75s ago 9m 48.1M 4273M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (8m) 75s ago 8m 49.6M 4273M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (6m) 75s ago 8m 69.7M - 514e6a882f6e ccdbacac91c5 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (6m) 2m ago 6m 71.2M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:02:13.654 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (6m) 75s ago 6m 74.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:02:13.654 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:02:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:13.655 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:13.655 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:14.205 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:02:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:14 smithi018 ceph-mon[23740]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:02:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:14 smithi018 ceph-mon[23740]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:14 smithi018 ceph-mon[28324]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:02:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:14 smithi018 ceph-mon[28324]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:14 smithi031 ceph-mon[24497]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:02:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:14 smithi031 ceph-mon[24497]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T00:02:16.331 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:02:16.332 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:02:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[23740]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3815889996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[28324]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:16.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:16 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3815889996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:16.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph -s' 2024-09-21T00:02:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:16 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:16 smithi031 ceph-mon[24497]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:16 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3815889996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:18.954 INFO:teuthology.orchestra.run.smithi018.stdout: cluster: 2024-09-21T00:02:18.954 INFO:teuthology.orchestra.run.smithi018.stdout: id: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:02:18.954 INFO:teuthology.orchestra.run.smithi018.stdout: health: HEALTH_OK 2024-09-21T00:02:18.954 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: services: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: mgr: y(active, since 8m), standbys: x 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: data: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: pools: 6 pools, 161 pgs 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: objects: 241 objects, 585 KiB 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: pgs: 161 active+clean 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: io: 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-21T00:02:18.955 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:02:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:18 smithi018 ceph-mon[23740]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:19.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:19] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-09-21T00:02:19.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:18 smithi018 ceph-mon[28324]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:18 smithi031 ceph-mon[24497]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:19.573 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:02:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:19 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1909562678' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:02:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:19 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1909562678' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:02:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:19 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1909562678' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:02:21.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:20 smithi018 ceph-mon[23740]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:21.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:20 smithi018 ceph-mon[28324]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:20 smithi031 ceph-mon[24497]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:21.591 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T00:02:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:21 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1973326706' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:02:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:21 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1973326706' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:02:22.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-09-21T00:02:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:21 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1973326706' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:02:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:22 smithi018 ceph-mon[23740]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:22 smithi018 ceph-mon[28324]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:22 smithi031 ceph-mon[24497]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:23.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:02:23.769 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:02:23.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:23.769 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:23.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:23.769 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:23.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:24.275 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:02:24.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph mgr fail' 2024-09-21T00:02:25.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:24 smithi018 ceph-mon[23740]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:25.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:24 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2034559337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:24 smithi018 ceph-mon[28324]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:24 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2034559337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:24 smithi031 ceph-mon[24497]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:02:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:24 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2034559337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:02:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:26 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:26.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:26 smithi018 ceph-mon[23740]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:26 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:26.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:26 smithi018 ceph-mon[28324]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:26 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:27.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:26 smithi031 ceph-mon[24497]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:02:27.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:02:27.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-mon[23740]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-mon[28324]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:27.680+0000 7fa964c23700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ignoring --setgroup ceph since I am not root 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-mgr[23971]: -- 172.21.15.18:0/873463223 <== mon.1 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621443b2340 con 0x56214438ac00 2024-09-21T00:02:27.926 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:27.867+0000 7f9d6623b000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:02:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:27 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:02:28.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:27 smithi031 ceph-mon[24497]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T00:02:28.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:27] ENGINE Bus STOPPING 2024-09-21T00:02:28.240 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:27.951+0000 7f9d6623b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:02:28.240 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:28.144+0000 7f9d6623b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:02:28.367 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-21T00:02:28.458 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:02:28.458 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:28] ENGINE Bus STOPPED 2024-09-21T00:02:28.458 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:28] ENGINE Bus STARTING 2024-09-21T00:02:28.458 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:28] ENGINE Serving on http://:::9283 2024-09-21T00:02:28.458 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:02:28] ENGINE Bus STARTED 2024-09-21T00:02:28.623 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:28.239+0000 7f9d6623b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:02:28.623 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:28.379+0000 7f9d6623b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:02:28.623 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:28.621+0000 7f9d6623b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:02:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:02:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: mgrmap e22: x(active, starting, since 0.833602s) 2024-09-21T00:02:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:02:28.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: Manager daemon x is now available 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:28.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:28 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:02:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: mgrmap e22: x(active, starting, since 0.833602s) 2024-09-21T00:02:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:02:28.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: Manager daemon x is now available 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:28.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3517616135' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: mgrmap e22: x(active, starting, since 0.833602s) 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:02:28.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: Manager daemon x is now available 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:02:28.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:28.930 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:02:29.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:28 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:28.982+0000 7f9d6623b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:02:29.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:29.077+0000 7f9d6623b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:02:29.252 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:29.250+0000 7f9d6623b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:02:30.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Queued rgw.foo for migration 2024-09-21T00:02:30.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-21T00:02:30.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-21T00:02:30.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Checking for cert/key for grafana.a 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Deploying cephadm binary to smithi018 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: mgrmap e23: x(active, since 1.84027s) 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: Deploying cephadm binary to smithi031 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:29 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Queued rgw.foo for migration 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Checking for cert/key for grafana.a 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Deploying cephadm binary to smithi018 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: mgrmap e23: x(active, since 1.84027s) 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: Deploying cephadm binary to smithi031 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Queued rgw.foo for migration 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Checking for cert/key for grafana.a 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Deploying cephadm binary to smithi018 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: mgrmap e23: x(active, since 1.84027s) 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: Deploying cephadm binary to smithi031 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:29 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:30.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:30 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:30.395+0000 7f9d6623b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:02:30.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:30 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:30.491+0000 7f9d6623b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:02:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:30 smithi031 ceph-mon[24497]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:30 smithi018 ceph-mon[23740]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:30 smithi018 ceph-mon[28324]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:31.176 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:30 smithi018 sudo[55809]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:31.176 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:30 smithi018 sudo[55809]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:31.176 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:30 smithi018 sudo[55809]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:31.176 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:30 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:30.783Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:02:31.543 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55892]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:31.544 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55892]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:31.544 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55892]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:02:31.425+0000 7f3e85873640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:00:02:31] ENGINE Error in HTTPServer.serve 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._connections.run(self.expiration_interval) 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:02:31.698 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._run(expiration_interval) 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: s = self.context.wrap_socket( 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.sslsocket_class._create( 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.do_handshake() 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._sslobj.do_handshake() 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:02:31.699 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:02:31.806 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:31.542+0000 7f9d6623b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:02:31.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-mon[23740]: mgrmap e24: x(active, since 3s) 2024-09-21T00:02:31.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:31.807 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-mon[28324]: mgrmap e24: x(active, since 3s) 2024-09-21T00:02:31.807 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:31.807 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55991]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:31.807 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55991]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:31.807 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:31 smithi018 sudo[55991]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:32.000 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-mon[24497]: mgrmap e24: x(active, since 3s) 2024-09-21T00:02:32.000 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:31 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:32.111 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:31.874+0000 7f9d6623b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:02:32.111 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.016+0000 7f9d6623b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:02:32.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.109+0000 7f9d6623b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:02:32.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.223+0000 7f9d6623b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:02:32.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.321+0000 7f9d6623b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:02:32.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56020]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:32.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56020]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:32.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56020]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:32.690 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.482+0000 7f9d6623b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:02:32.690 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.583+0000 7f9d6623b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:02:32.691 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56073]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:32.691 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56073]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:32.691 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:32 smithi018 sudo[56073]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: [21/Sep/2024:00:02:31] ENGINE Bus STARTING 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: [21/Sep/2024:00:02:31] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: [21/Sep/2024:00:02:31] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: [21/Sep/2024:00:02:31] ENGINE Bus STARTED 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: [21/Sep/2024:00:02:31] ENGINE Error in HTTPServer.serve 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:02:32.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._run(expiration_interval) 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: s = self.context.wrap_socket( 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.sslsocket_class._create( 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.do_handshake() 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:02:32.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._sslobj.do_handshake() 2024-09-21T00:02:32.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:02:32.914 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:32 smithi031 ceph-mon[24497]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:33.008 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:32.689+0000 7f9d6623b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:33.007+0000 7f9d6623b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: [21/Sep/2024:00:02:31] ENGINE Bus STARTING 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: [21/Sep/2024:00:02:31] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: [21/Sep/2024:00:02:31] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: [21/Sep/2024:00:02:31] ENGINE Bus STARTED 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: [21/Sep/2024:00:02:31] ENGINE Error in HTTPServer.serve 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:02:33.009 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:02:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.do_handshake() 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[28324]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: [21/Sep/2024:00:02:31] ENGINE Bus STARTING 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: [21/Sep/2024:00:02:31] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: [21/Sep/2024:00:02:31] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: [21/Sep/2024:00:02:31] ENGINE Bus STARTED 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: [21/Sep/2024:00:02:31] ENGINE Error in HTTPServer.serve 2024-09-21T00:02:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:02:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:02:33.013 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.do_handshake() 2024-09-21T00:02:33.013 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:02:33.013 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:02:33.013 INFO:journalctl@ceph.mon.a.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:02:33.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:32 smithi018 ceph-mon[23740]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:33.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:33] "GET /metrics HTTP/1.1" 200 34891 "" "Prometheus/2.33.4" 2024-09-21T00:02:33.332 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:33.101+0000 7f9d6623b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:02:33.332 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:33.197+0000 7f9d6623b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:02:33.333 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56127]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:33.333 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56127]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:33.333 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56127]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:33.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56149]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:33.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56149]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:33.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:33 smithi018 sudo[56149]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:33.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:02:33.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:02:33.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:02:33.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:33 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:33.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:34.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:34 smithi018 sudo[56185]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:34.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:34 smithi018 sudo[56185]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:34.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:02:34 smithi018 sudo[56185]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:34.359+0000 7f9d6623b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [21/Sep/2024:00:02:34] ENGINE Bus STARTING 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: CherryPy Checker: 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: The Application mounted at '' has an empty config. 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [21/Sep/2024:00:02:34] ENGINE Serving on http://:::9283 2024-09-21T00:02:34.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: [21/Sep/2024:00:02:34] ENGINE Bus STARTED 2024-09-21T00:02:34.751 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46495]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:34.751 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46495]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:34.752 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46495]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:34.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:34.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:34.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:34.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: Standby manager daemon y started 2024-09-21T00:02:34.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:02:34.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:02:34.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:02:34.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:02:34.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:34.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:34 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:34.787Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: Standby manager daemon y started 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: Standby manager daemon y started 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1804097119' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:34 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:35.251 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46654]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:35.251 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46654]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:35.251 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:02:34 smithi031 sudo[46654]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:35.752 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[46895]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:35.752 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[46895]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:35.752 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[46895]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi018 to 2993M 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: mgrmap e25: x(active, since 8s), standbys: y 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi018 to 2993M 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: mgrmap e25: x(active, since 8s), standbys: y 2024-09-21T00:02:36.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:35 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:02:36.251 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[47294]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:36.251 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[47294]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:36.251 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:02:35 smithi031 sudo[47294]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi018 to 2993M 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: mgrmap e25: x(active, since 8s), standbys: y 2024-09-21T00:02:36.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:35 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:02:36.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47553]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:36.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47553]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:36.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47553]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:36.894 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:36 smithi018 systemd[1]: Stopping Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:02:37.175 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 podman[57402]: 2024-09-21 00:02:37.010038402 +0000 UTC m=+0.131634903 container died b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Adjusting osd_memory_target on smithi031 to 3249M 2024-09-21T00:02:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:02:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:02:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 56 KiB/s wr, 23 op/s 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Adjusting osd_memory_target on smithi031 to 3249M 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 56 KiB/s wr, 23 op/s 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:37.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:36 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.231 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47556]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:37.231 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47556]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:37.231 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:02:36 smithi031 sudo[47556]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:37.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Adjusting osd_memory_target on smithi031 to 3249M 2024-09-21T00:02:37.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:02:37.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:02:37.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 56 KiB/s wr, 23 op/s 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.232 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.233 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.233 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.233 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:36 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:37.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47559]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:37.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47559]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:37.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47559]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:37.642 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 podman[57402]: 2024-09-21 00:02:37.407296536 +0000 UTC m=+0.528893028 container cleanup b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:37.642 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 bash[57402]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a 2024-09-21T00:02:37.920 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 podman[57414]: 2024-09-21 00:02:37.641708561 +0000 UTC m=+0.634884758 container remove b8ef2efc32f5a0878ece8a2685b9d515c879179ef4a2cb7ec0dd7f0aee6afcf4 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:37.921 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:02:38.001 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47562]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:38.001 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47562]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:38.001 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:02:37 smithi031 sudo[47562]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:38.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57496]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:38.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57496]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:38.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57496]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:38.365 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T00:02:38.365 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 systemd[1]: Stopped Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:02:38.365 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:37 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Consumed 4.572s CPU time. 2024-09-21T00:02:38.365 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 systemd[1]: Starting Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:02:38.615 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 bash[57545]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T00:02:38.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57558]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:38.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57558]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:38.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:38 smithi018 sudo[57558]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:38.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 bash[57545]: Getting image source signatures 2024-09-21T00:02:38.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 bash[57545]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T00:02:38.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 bash[57545]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T00:02:38.926 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:38 smithi018 bash[57545]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T00:02:39.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:02:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57573]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57573]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57573]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[23740]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[23740]: Deploying daemon node-exporter.a on smithi018 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57609]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57609]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:39.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 sudo[57609]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:39.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[28324]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T00:02:39.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[28324]: Deploying daemon node-exporter.a on smithi018 2024-09-21T00:02:39.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:39 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:39.990 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:39 smithi031 ceph-mon[24497]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T00:02:39.990 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:39 smithi031 ceph-mon[24497]: Deploying daemon node-exporter.a on smithi018 2024-09-21T00:02:39.990 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:39 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:39 smithi031 sudo[47565]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T00:02:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:39 smithi031 sudo[47565]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 sudo[47565]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:40.341 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 bash[57545]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T00:02:40.675 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 bash[57545]: Writing manifest to image destination 2024-09-21T00:02:40.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 44 KiB/s wr, 18 op/s 2024-09-21T00:02:40.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 71 KiB/s wr, 17 op/s 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 44 KiB/s wr, 18 op/s 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 71 KiB/s wr, 17 op/s 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 sudo[47568]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T00:02:40.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 sudo[47568]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 sudo[47568]: pam_unix(sudo:session): session closed for user root 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 44 KiB/s wr, 18 op/s 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T00:02:40.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 71 KiB/s wr, 17 op/s 2024-09-21T00:02:40.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T00:02:40.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:02:40.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:02:40.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:40 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 podman[57545]: 2024-09-21 00:02:40.596906529 +0000 UTC m=+2.332452097 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 podman[57545]: 2024-09-21 00:02:40.720289197 +0000 UTC m=+2.455834810 container create b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 podman[57545]: 2024-09-21 00:02:40.923557991 +0000 UTC m=+2.659103570 container init b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 podman[57545]: 2024-09-21 00:02:40.927084549 +0000 UTC m=+2.662630123 container start b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.930Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.930Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.931Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T00:02:41.008 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.931Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.931Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.931Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T00:02:41.009 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T00:02:41.010 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T00:02:41.011 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T00:02:41.012 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T00:02:41.013 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T00:02:41.014 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.932Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T00:02:41.015 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.933Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T00:02:41.015 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a[57627]: ts=2024-09-21T00:02:40.933Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T00:02:41.015 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:41 smithi018 bash[57545]: b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 2024-09-21T00:02:41.307 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:02:41 smithi018 systemd[1]: Started Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:02:42.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:42 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:42 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:42 smithi031 ceph-mon[24497]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T00:02:42.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:42 smithi031 ceph-mon[24497]: Deploying daemon alertmanager.a on smithi018 2024-09-21T00:02:42.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:42 smithi031 ceph-mon[24497]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[23740]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[23740]: Deploying daemon alertmanager.a on smithi018 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[23740]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:42.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[28324]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T00:02:42.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[28324]: Deploying daemon alertmanager.a on smithi018 2024-09-21T00:02:42.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:42 smithi018 ceph-mon[28324]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:43.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:43.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:43] "GET /metrics HTTP/1.1" 200 34891 "" "Prometheus/2.33.4" 2024-09-21T00:02:43.523 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:43.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:43.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=error ts=2024-09-21T00:02:43.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:43.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:43.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.18:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.18 because it doesn't contain any IP SANs" 2024-09-21T00:02:43.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:43 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=warn ts=2024-09-21T00:02:43.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.31:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.31 because it doesn't contain any IP SANs" 2024-09-21T00:02:44.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:44 smithi018 ceph-mon[23740]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:44.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:44 smithi018 ceph-mon[28324]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:44.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:44 smithi031 ceph-mon[24497]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 83 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 65 KiB/s wr, 16 op/s 2024-09-21T00:02:46.770 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:46 smithi018 ceph-mon[23740]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 87 KiB/s wr, 18 op/s 2024-09-21T00:02:46.770 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:46 smithi018 ceph-mon[23740]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:46.770 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:46 smithi018 ceph-mon[28324]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 87 KiB/s wr, 18 op/s 2024-09-21T00:02:46.770 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:46 smithi018 ceph-mon[28324]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:46 smithi031 ceph-mon[24497]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 87 KiB/s wr, 18 op/s 2024-09-21T00:02:47.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:46 smithi031 ceph-mon[24497]: from='client.24763 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:02:48.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:47 smithi018 systemd[1]: Stopping Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:02:48.510 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[48228]: level=info ts=2024-09-21T00:02:48.217Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:02:48.510 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 podman[57984]: 2024-09-21 00:02:48.238621268 +0000 UTC m=+0.132454115 container died 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:48.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:48 smithi018 ceph-mon[23740]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:48.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:48 smithi018 ceph-mon[28324]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:48.894 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 podman[57984]: 2024-09-21 00:02:48.600664853 +0000 UTC m=+0.494497685 container cleanup 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:48.894 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 bash[57984]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a 2024-09-21T00:02:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:48 smithi031 ceph-mon[24497]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:49.169 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:49 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:02:49.169 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 podman[57996]: 2024-09-21 00:02:48.893303662 +0000 UTC m=+0.649082548 container remove 11c7d61a9d987b90e30d9c9e8cd6d312788faf2e0c7aeb924873baa58270039e (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:49.169 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:48 smithi018 podman[57996]: 2024-09-21 00:02:48.968893598 +0000 UTC m=+0.724672483 volume remove 0c4bbed7491e059da659b0804639f1504fdde5f3001d52a53607573c3b9b7be7 2024-09-21T00:02:49.613 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T00:02:49.614 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 systemd[1]: Stopped Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:02:49.614 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service: Consumed 1.511s CPU time. 2024-09-21T00:02:49.614 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 systemd[1]: Starting Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:02:49.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 podman[58126]: 2024-09-21 00:02:49.516313291 +0000 UTC m=+0.016525601 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T00:02:49.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 podman[58126]: 2024-09-21 00:02:49.630796018 +0000 UTC m=+0.131008371 volume create 32167432dfd5ec252296abcbb8161d1a64627eb3f1445f1566a6738b49db906d 2024-09-21T00:02:49.925 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:49 smithi018 podman[58126]: 2024-09-21 00:02:49.731047334 +0000 UTC m=+0.231259644 container create 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 podman[58126]: 2024-09-21 00:02:50.025589516 +0000 UTC m=+0.525801826 container init 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 podman[58126]: 2024-09-21 00:02:50.028644295 +0000 UTC m=+0.528856605 container start 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:02:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.040Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T00:02:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.040Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T00:02:50.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.042Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.043Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.066Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.066Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.069Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:50.069Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 bash[58126]: 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 2024-09-21T00:02:50.426 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:50 smithi018 systemd[1]: Started Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:02:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:51.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:50 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 54 KiB/s wr, 5 op/s 2024-09-21T00:02:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:02:51.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:50 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:02:52.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:51 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:02:52.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:51 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:02:52.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:51 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:02:52.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:51 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:02:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:51 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:02:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:51 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:02:52.372 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:02:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:02:52.044Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000600724s 2024-09-21T00:02:52.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:02:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.31:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.18:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:02:53.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:52 smithi018 ceph-mon[28324]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:53.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:52 smithi018 ceph-mon[23740]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:53.110 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:52 smithi031 ceph-mon[24497]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:53.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:02:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:53] "GET /metrics HTTP/1.1" 200 37883 "" "Prometheus/2.33.4" 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[23740]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1425377282' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[28324]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:54 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1425377282' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:02:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:54 smithi031 ceph-mon[24497]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-21T00:02:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:54 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:54 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:02:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:54 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1425377282' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[23740]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[23740]: Deploying daemon prometheus.a on smithi031 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2215438832' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[28324]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[28324]: Deploying daemon prometheus.a on smithi031 2024-09-21T00:02:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2215438832' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:55 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:55 smithi031 ceph-mon[24497]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:02:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:55 smithi031 ceph-mon[24497]: Deploying daemon prometheus.a on smithi031 2024-09-21T00:02:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2215438832' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:55 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]: dispatch 2024-09-21T00:02:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[23740]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-21T00:02:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]': finished 2024-09-21T00:02:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[23740]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T00:02:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1557870127' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[28324]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-21T00:02:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]': finished 2024-09-21T00:02:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[28324]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T00:02:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1557870127' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:57.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:56 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:56 smithi031 ceph-mon[24497]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-21T00:02:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2820254179"}]': finished 2024-09-21T00:02:57.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:56 smithi031 ceph-mon[24497]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T00:02:57.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:56 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1557870127' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:57.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:56 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]: dispatch 2024-09-21T00:02:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]': finished 2024-09-21T00:02:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[23740]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T00:02:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]: dispatch 2024-09-21T00:02:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]': finished 2024-09-21T00:02:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[28324]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T00:02:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:57 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]: dispatch 2024-09-21T00:02:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:57 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3798661254"}]': finished 2024-09-21T00:02:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:57 smithi031 ceph-mon[24497]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T00:02:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:57 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]: dispatch 2024-09-21T00:02:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:02:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]': finished 2024-09-21T00:02:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T00:02:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4009184904' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]': finished 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4009184904' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:58 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:02:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1932543597' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/269195622"}]': finished 2024-09-21T00:02:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T00:02:59.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4009184904' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]: dispatch 2024-09-21T00:02:59.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:58 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:02:59.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[23967]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:02:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-21T00:03:00.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]': finished 2024-09-21T00:03:00.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[23740]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T00:03:00.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/40180208' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]': finished 2024-09-21T00:03:00.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[28324]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T00:03:00.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/40180208' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:02:59 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:59 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1680364751"}]': finished 2024-09-21T00:03:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:59 smithi031 ceph-mon[24497]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T00:03:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:59 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/40180208' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:02:59 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]: dispatch 2024-09-21T00:03:00.425 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:03:00.046Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003216434s 2024-09-21T00:03:01.152 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:00 smithi031 ceph-mon[24497]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:01.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:00 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]': finished 2024-09-21T00:03:01.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:00 smithi031 ceph-mon[24497]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T00:03:01.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:00 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2416073200' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:01.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:00 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[28324]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]': finished 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[28324]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2416073200' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[23740]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/269195622"}]': finished 2024-09-21T00:03:01.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[23740]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T00:03:01.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2416073200' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:01.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:00 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]: dispatch 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]': finished 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[23740]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]: dispatch 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]': finished 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[28324]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T00:03:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:01 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]: dispatch 2024-09-21T00:03:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:01 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2592259683"}]': finished 2024-09-21T00:03:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:01 smithi031 ceph-mon[24497]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T00:03:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:01 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]: dispatch 2024-09-21T00:03:02.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.31:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.18:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:03.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:02 smithi031 ceph-mon[24497]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:03.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:02 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]': finished 2024-09-21T00:03:03.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:02 smithi031 ceph-mon[24497]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[23740]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]': finished 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[23740]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[28324]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4025529414' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1401144154"}]': finished 2024-09-21T00:03:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:02 smithi018 ceph-mon[28324]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T00:03:03.414 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:03] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-09-21T00:03:04.810 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 systemd[1]: Stopping Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:05.065 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-mon[24497]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.821Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.821Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-21T00:03:05.065 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.822Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.825Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.825Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[41414]: ts=2024-09-21T00:03:04.825Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-21T00:03:05.066 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:04 smithi031 podman[48030]: 2024-09-21 00:03:04.832161726 +0000 UTC m=+0.153188101 container died ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:04 smithi018 ceph-mon[23740]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:03:05.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:04 smithi018 ceph-mon[28324]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:03:05.339 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 podman[48030]: 2024-09-21 00:03:05.171798617 +0000 UTC m=+0.492824990 container cleanup ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:05.340 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 bash[48030]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-21T00:03:05.606 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 podman[48044]: 2024-09-21 00:03:05.339189667 +0000 UTC m=+0.502505450 container remove ccdbacac91c5834f61063ba0735287750497a4d8f2a0faf93daf8b348c4b32f0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:05 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:06.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T00:03:06.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 systemd[1]: Stopped Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:06.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Consumed 2.438s CPU time. 2024-09-21T00:03:06.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:05 smithi031 systemd[1]: Starting Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:05 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:05 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 podman[48169]: 2024-09-21 00:03:05.93758606 +0000 UTC m=+0.017937172 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 podman[48169]: 2024-09-21 00:03:06.048795244 +0000 UTC m=+0.129146329 container create 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 podman[48169]: 2024-09-21 00:03:06.225948906 +0000 UTC m=+0.306300011 container init 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 podman[48169]: 2024-09-21 00:03:06.228704946 +0000 UTC m=+0.309056032 container start 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.252Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.253Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.253Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi031 (none))" 2024-09-21T00:03:06.303 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.253Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.253Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.254Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.254Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.256Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.256Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.257Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.259Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.56382ms 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.259Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T00:03:06.304 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.264Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.279Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.279Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.279Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.866µs wal_replay_duration=20.293389ms wbl_replay_duration=168ns total_replay_duration=21.926123ms 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.283Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.283Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.283Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.302Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.598308ms db_storage=818ns remote_storage=1.06µs web_handler=425ns query_engine=664ns scrape=447.254µs scrape_sd=90.31µs notify=11.386µs notify_sd=14.322µs rules=17.706796ms tracing=4.849µs 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.302Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T00:03:06.305 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:06.302Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T00:03:06.734 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 bash[48169]: 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 2024-09-21T00:03:06.734 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:06 smithi031 systemd[1]: Started Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-mon[24497]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:06 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.002 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:06 smithi031 systemd[1]: Stopping Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[23740]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[28324]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:06 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:07.449 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 podman[48348]: 2024-09-21 00:03:07.191058871 +0000 UTC m=+0.115161303 container died 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:07.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 podman[48348]: 2024-09-21 00:03:07.54701741 +0000 UTC m=+0.471119842 container cleanup 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:07.752 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 bash[48348]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b 2024-09-21T00:03:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:07 smithi018 ceph-mon[28324]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T00:03:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:07 smithi018 ceph-mon[28324]: Deploying daemon node-exporter.b on smithi031 2024-09-21T00:03:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:07 smithi018 ceph-mon[23740]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T00:03:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:07 smithi018 ceph-mon[23740]: Deploying daemon node-exporter.b on smithi031 2024-09-21T00:03:08.181 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:07 smithi031 ceph-mon[24497]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T00:03:08.181 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:07 smithi031 ceph-mon[24497]: Deploying daemon node-exporter.b on smithi031 2024-09-21T00:03:08.181 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 podman[48361]: 2024-09-21 00:03:07.823111882 +0000 UTC m=+0.635398334 container remove 95837fdf8e6da3f894158edfdc7a68eb3e188c25ce5af92a18e34644231233d5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:08.181 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:03:08.182 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T00:03:08.182 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 systemd[1]: Stopped Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:08.182 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:07 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Consumed 4.571s CPU time. 2024-09-21T00:03:08.501 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 systemd[1]: Starting Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:08.501 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 bash[48490]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T00:03:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:08 smithi031 ceph-mon[24497]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-09-21T00:03:09.001 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 bash[48490]: Getting image source signatures 2024-09-21T00:03:09.001 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 bash[48490]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T00:03:09.001 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 bash[48490]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T00:03:09.001 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:08 smithi031 bash[48490]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T00:03:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:08 smithi018 ceph-mon[23740]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-09-21T00:03:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:08 smithi018 ceph-mon[28324]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-09-21T00:03:10.251 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:09 smithi031 bash[48490]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T00:03:10.251 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 bash[48490]: Writing manifest to image destination 2024-09-21T00:03:10.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 podman[48490]: 2024-09-21 00:03:10.385016322 +0000 UTC m=+2.107277890 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T00:03:10.751 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 podman[48490]: 2024-09-21 00:03:10.491457644 +0000 UTC m=+2.213719193 container create 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:10 smithi018 ceph-mon[23740]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:03:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:10 smithi018 ceph-mon[28324]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:03:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-mon[24497]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:03:11.251 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 podman[48490]: 2024-09-21 00:03:10.803461616 +0000 UTC m=+2.525723168 container init 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:11.251 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 podman[48490]: 2024-09-21 00:03:10.806773868 +0000 UTC m=+2.529035418 container start 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.811Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.811Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.811Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.811Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.812Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.812Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.812Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T00:03:11.252 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T00:03:11.253 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T00:03:11.254 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T00:03:11.255 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b[48565]: ts=2024-09-21T00:03:10.813Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 bash[48490]: 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 2024-09-21T00:03:11.256 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:03:10 smithi031 systemd[1]: Started Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.362 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.363 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.363 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.363 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: Reconfiguring daemon grafana.a on smithi031 2024-09-21T00:03:12.363 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:12 smithi031 ceph-mon[24497]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: Reconfiguring daemon grafana.a on smithi031 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[28324]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: Reconfiguring daemon grafana.a on smithi031 2024-09-21T00:03:12.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:12 smithi018 ceph-mon[23740]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:03:13.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:13 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:13.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:13 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:13 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:13] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.43.0" 2024-09-21T00:03:14.056 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:13 smithi031 systemd[1]: Stopping Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:14 smithi031 ceph-mon[24497]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-21T00:03:14.501 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-21T00:03:14+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-21T00:03:14.501 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[40633]: t=2024-09-21T00:03:14+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2024-09-21T00:03:14.501 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 podman[48739]: 2024-09-21 00:03:14.076660828 +0000 UTC m=+0.223313569 container died 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, description=Ceph Grafana Container, release=236.1648460182, distribution-scope=public, io.openshift.tags=base rhel8, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-type=git) 2024-09-21T00:03:14.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:14 smithi018 ceph-mon[23740]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-21T00:03:14.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:14 smithi018 ceph-mon[28324]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-09-21T00:03:14.949 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 podman[48739]: 2024-09-21 00:03:14.630530107 +0000 UTC m=+0.777182836 container cleanup 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, vcs-type=git, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, name=ubi8, vendor=Red Hat, Inc., release=236.1648460182) 2024-09-21T00:03:14.949 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 bash[48739]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a 2024-09-21T00:03:14.949 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 bash[48766]: Error: no container with name or ID "ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana.a" found: no such container 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 podman[48751]: 2024-09-21 00:03:14.948536971 +0000 UTC m=+0.872816601 container remove 9103855ea14ba117e3fc612b1e92b4ba74254d4fbd1862da4cfa45c2107a2a69 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.openshift.expose-services=, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, name=ubi8, architecture=x86_64, vcs-type=git) 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 systemd[1]: Stopped Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Consumed 3.406s CPU time. 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:14 smithi031 systemd[1]: Starting Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 podman[48827]: 2024-09-21 00:03:15.094751201 +0000 UTC m=+0.016160886 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-21T00:03:15.200 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 podman[48827]: 2024-09-21 00:03:15.199407787 +0000 UTC m=+0.120817464 container create 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, name=ubi8, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, architecture=x86_64, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182) 2024-09-21T00:03:15.450 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:15.450 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 podman[48827]: 2024-09-21 00:03:15.418618263 +0000 UTC m=+0.340027957 container init 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , description=Ceph Grafana Container, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, release=236.1648460182, vcs-type=git, distribution-scope=public) 2024-09-21T00:03:15.450 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 podman[48827]: 2024-09-21 00:03:15.421488599 +0000 UTC m=+0.342898274 container start 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, name=ubi8, io.buildah.version=1.24.2, architecture=x86_64, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=236.1648460182, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-09-21T00:03:15.594 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:15 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:15.594 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:15 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:15.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Bus STOPPING 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T00:03:15.752 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="App mode production" logger=settings 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T00:03:15.753 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=550.334µs 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 bash[48827]: 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb 2024-09-21T00:03:15.754 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 systemd[1]: Started Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-21T00:03:15.755 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:03:15+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T00:03:16.232 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:03:16.233 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Bus STOPPED 2024-09-21T00:03:16.233 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Bus STARTING 2024-09-21T00:03:16.233 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Serving on http://:::9283 2024-09-21T00:03:16.233 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Bus STARTED 2024-09-21T00:03:16.233 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:15] ENGINE Bus STOPPING 2024-09-21T00:03:16.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:03:16.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE Bus STOPPED 2024-09-21T00:03:16.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE Bus STARTING 2024-09-21T00:03:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[23740]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:16.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:16.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:16.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:16 smithi018 ceph-mon[28324]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:17.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE Serving on http://:::9283 2024-09-21T00:03:17.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE Bus STARTED 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:16] ENGINE Bus STOPPING 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: Adding iSCSI gateway http://:@172.21.15.18:5000 to Dashboard 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi018"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:03:17.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi031.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi031.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:17.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:16 smithi031 ceph-mon[24497]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:17.399 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:03:17.400 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:17] ENGINE Bus STOPPED 2024-09-21T00:03:17.400 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:17] ENGINE Bus STARTING 2024-09-21T00:03:17.400 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:17] ENGINE Serving on http://:::9283 2024-09-21T00:03:17.400 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:03:17] ENGINE Bus STARTED 2024-09-21T00:03:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:18 smithi018 ceph-mon[23740]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:18 smithi018 ceph-mon[28324]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:18 smithi031 ceph-mon[24497]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:19.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[28324]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:20.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:20.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:20 smithi018 ceph-mon[23740]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:20 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:20 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:20 smithi031 ceph-mon[24497]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:03:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:03:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:03:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:03:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:21 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:21 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:21 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:21 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:03:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:21 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:03:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:21 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:03:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:22 smithi031 ceph-mon[24497]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:22 smithi018 ceph-mon[23740]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:23.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:22 smithi018 ceph-mon[28324]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:23.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:23] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.43.0" 2024-09-21T00:03:23.754 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:23.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:24 smithi031 ceph-mon[24497]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:25.599 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:24 smithi018 ceph-mon[23740]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:25.600 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:24 smithi018 ceph-mon[28324]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:26 smithi031 ceph-mon[24497]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:27.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:26 smithi018 ceph-mon[28324]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:27.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:26 smithi018 ceph-mon[23740]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:28.962 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:28 smithi031 ceph-mon[24497]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:28 smithi018 ceph-mon[28324]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:28 smithi018 ceph-mon[23740]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:30 smithi031 ceph-mon[24497]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:30 smithi018 ceph-mon[23740]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:30 smithi018 ceph-mon[28324]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:32 smithi031 ceph-mon[24497]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:33.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:32 smithi018 ceph-mon[23740]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:32 smithi018 ceph-mon[28324]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:33] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.43.0" 2024-09-21T00:03:33.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:34 smithi031 ceph-mon[24497]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:34 smithi018 ceph-mon[23740]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:34 smithi018 ceph-mon[28324]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:36 smithi031 ceph-mon[24497]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:36 smithi018 ceph-mon[23740]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:36 smithi018 ceph-mon[28324]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:38.977 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:38 smithi031 ceph-mon[24497]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:38 smithi018 ceph-mon[23740]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:38 smithi018 ceph-mon[28324]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:41.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:40 smithi018 ceph-mon[23740]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:41.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:40 smithi018 ceph-mon[28324]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:40 smithi031 ceph-mon[24497]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:42 smithi031 ceph-mon[24497]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:42 smithi018 ceph-mon[23740]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:42 smithi018 ceph-mon[28324]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:43] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.43.0" 2024-09-21T00:03:43.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:44.357 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:44.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:44 smithi031 ceph-mon[24497]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:44 smithi018 ceph-mon[23740]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:44 smithi018 ceph-mon[28324]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:45 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:45 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:45 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:46 smithi031 ceph-mon[24497]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:46 smithi018 ceph-mon[23740]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:46 smithi018 ceph-mon[28324]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:48.995 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:48 smithi031 ceph-mon[24497]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:48 smithi018 ceph-mon[23740]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:48 smithi018 ceph-mon[28324]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:50 smithi031 ceph-mon[24497]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:50 smithi018 ceph-mon[23740]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:50 smithi018 ceph-mon[28324]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:53.351 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:53 smithi031 ceph-mon[24497]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:53 smithi018 ceph-mon[23740]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:53 smithi018 ceph-mon[28324]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:03:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:03:53] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.43.0" 2024-09-21T00:03:53.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:55 smithi018 ceph-mon[23740]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:55 smithi018 ceph-mon[28324]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:55 smithi031 ceph-mon[24497]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:03:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:56 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:56.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:56 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:56 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:03:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:57 smithi018 ceph-mon[23740]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:57.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:57 smithi018 ceph-mon[28324]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:57 smithi031 ceph-mon[24497]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:59.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:03:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:03:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:03:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:59 smithi018 ceph-mon[23740]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:03:59 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:59 smithi018 ceph-mon[28324]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:03:59 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:03:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:59 smithi031 ceph-mon[24497]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:03:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:03:59 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:01.303 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:01 smithi031 ceph-mon[24497]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:01 smithi018 ceph-mon[23740]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:01 smithi018 ceph-mon[28324]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:03.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:03 smithi031 ceph-mon[24497]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:03 smithi018 ceph-mon[23740]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:03 smithi018 ceph-mon[28324]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:03] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.43.0" 2024-09-21T00:04:03.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:03.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:05 smithi018 ceph-mon[23740]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:05 smithi018 ceph-mon[28324]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:05 smithi031 ceph-mon[24497]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:06 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:06 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:06 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:07.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:07 smithi018 ceph-mon[23740]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:07.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:07 smithi018 ceph-mon[28324]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:07 smithi031 ceph-mon[24497]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:09.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:08 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:09 smithi018 ceph-mon[23740]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:09.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:09 smithi018 ceph-mon[28324]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:09 smithi031 ceph-mon[24497]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:11.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:11 smithi018 ceph-mon[23740]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:11.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:11 smithi018 ceph-mon[28324]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:11.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:11 smithi031 ceph-mon[24497]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:13.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:13 smithi031 ceph-mon[24497]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:13.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:13 smithi018 ceph-mon[23740]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:13.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:13 smithi018 ceph-mon[28324]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:13] "GET /metrics HTTP/1.1" 200 37883 "" "Prometheus/2.43.0" 2024-09-21T00:04:13.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:14.378 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:14 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:14.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:14 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:14 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:15.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:15 smithi018 ceph-mon[23740]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:15.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:15 smithi018 ceph-mon[28324]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:15.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:15 smithi031 ceph-mon[24497]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:16 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:16 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:16 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:17.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:17 smithi018 ceph-mon[23740]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:17.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:17 smithi018 ceph-mon[28324]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:17.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:17 smithi031 ceph-mon[24497]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:19.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:19 smithi018 ceph-mon[23740]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:19 smithi018 ceph-mon[28324]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:19.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:19 smithi031 ceph-mon[24497]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:20 smithi018 ceph-mon[23740]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:20 smithi018 ceph-mon[28324]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:20 smithi031 ceph-mon[24497]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:22 smithi018 ceph-mon[23740]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:22 smithi018 ceph-mon[28324]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:22 smithi031 ceph-mon[24497]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:23] "GET /metrics HTTP/1.1" 200 37883 "" "Prometheus/2.43.0" 2024-09-21T00:04:23.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:24 smithi031 ceph-mon[24497]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:24 smithi018 ceph-mon[23740]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:24 smithi018 ceph-mon[28324]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:04:26.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:04:26.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:25 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:04:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:04:26.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:25 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:04:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:26 smithi031 ceph-mon[24497]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:27.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:26 smithi018 ceph-mon[23740]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:28.364 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:26 smithi018 ceph-mon[28324]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:28.982 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:28 smithi031 ceph-mon[24497]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:28 smithi018 ceph-mon[23740]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:28 smithi018 ceph-mon[28324]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:30 smithi031 ceph-mon[24497]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:30 smithi018 ceph-mon[23740]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:31.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:30 smithi018 ceph-mon[28324]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:32 smithi031 ceph-mon[24497]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:33.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:32 smithi018 ceph-mon[23740]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:32 smithi018 ceph-mon[28324]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:33] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.43.0" 2024-09-21T00:04:33.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:34 smithi031 ceph-mon[24497]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:34 smithi018 ceph-mon[23740]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:34 smithi018 ceph-mon[28324]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:36 smithi031 ceph-mon[24497]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:36 smithi018 ceph-mon[23740]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:36 smithi018 ceph-mon[28324]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:38.997 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:38 smithi031 ceph-mon[24497]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:38 smithi018 ceph-mon[23740]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:38 smithi018 ceph-mon[28324]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:40 smithi031 ceph-mon[24497]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:40 smithi018 ceph-mon[23740]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:40 smithi018 ceph-mon[28324]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:43.351 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:43 smithi031 ceph-mon[24497]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:43 smithi018 ceph-mon[23740]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:43 smithi018 ceph-mon[28324]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:43] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-21T00:04:43.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:43.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:44.400 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:44 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:44.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:44 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:44 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:45 smithi018 ceph-mon[23740]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:45.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:45 smithi018 ceph-mon[28324]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:45 smithi031 ceph-mon[24497]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:46 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:46 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:46 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:47 smithi018 ceph-mon[23740]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:47 smithi018 ceph-mon[28324]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:47 smithi031 ceph-mon[24497]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:49.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:49 smithi018 ceph-mon[23740]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:49 smithi018 ceph-mon[28324]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:49 smithi031 ceph-mon[24497]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:51 smithi018 ceph-mon[23740]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:51 smithi018 ceph-mon[28324]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:51 smithi031 ceph-mon[24497]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:53.351 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:53 smithi031 ceph-mon[24497]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:53 smithi018 ceph-mon[23740]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:53 smithi018 ceph-mon[28324]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:04:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:04:53] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-21T00:04:53.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:55 smithi018 ceph-mon[28324]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:55 smithi018 ceph-mon[23740]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:55 smithi031 ceph-mon[24497]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:04:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:56 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:56.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:56 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:56 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:04:57.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:57 smithi018 ceph-mon[28324]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:57 smithi018 ceph-mon[23740]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:57 smithi031 ceph-mon[24497]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:59.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:04:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:04:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:04:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:59 smithi018 ceph-mon[23740]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:04:59 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:59 smithi018 ceph-mon[28324]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:04:59 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:04:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:59 smithi031 ceph-mon[24497]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:04:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:04:59 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:01.303 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:01 smithi031 ceph-mon[24497]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:01 smithi018 ceph-mon[23740]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:01.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:01 smithi018 ceph-mon[28324]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:03.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:03 smithi031 ceph-mon[24497]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:03 smithi018 ceph-mon[23740]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:03 smithi018 ceph-mon[28324]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:03] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-21T00:05:03.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:05 smithi018 ceph-mon[23740]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:05 smithi018 ceph-mon[28324]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:05 smithi031 ceph-mon[24497]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:06 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:06 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:06 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:07.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:07 smithi018 ceph-mon[23740]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:07.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:07 smithi018 ceph-mon[28324]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:07 smithi031 ceph-mon[24497]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:09.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:08 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:09.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:09 smithi018 ceph-mon[28324]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:09 smithi018 ceph-mon[23740]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:09 smithi031 ceph-mon[24497]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:11.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:11 smithi018 ceph-mon[23740]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:11.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:11 smithi018 ceph-mon[28324]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:11.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:11 smithi031 ceph-mon[24497]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:13.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:13 smithi031 ceph-mon[24497]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:13.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:13 smithi018 ceph-mon[23740]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:13.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:13 smithi018 ceph-mon[28324]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:13] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:05:13.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:14.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:14 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:14 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:14 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:15.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:15 smithi018 ceph-mon[23740]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:15.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:15 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:15.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:15 smithi018 ceph-mon[28324]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:15.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:15 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:15.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:15 smithi031 ceph-mon[24497]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:15.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:15 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:16 smithi031 ceph-mon[24497]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:16 smithi018 ceph-mon[23740]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:16 smithi018 ceph-mon[28324]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:18 smithi031 ceph-mon[24497]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:19.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:18 smithi018 ceph-mon[23740]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:18 smithi018 ceph-mon[28324]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:20 smithi018 ceph-mon[23740]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:20 smithi018 ceph-mon[28324]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:20 smithi031 ceph-mon[24497]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:22 smithi018 ceph-mon[23740]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:22 smithi018 ceph-mon[28324]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:22 smithi031 ceph-mon[24497]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:23] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:05:23.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:24 smithi018 ceph-mon[23740]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:24 smithi018 ceph-mon[28324]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:24 smithi031 ceph-mon[24497]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:27.123 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:26 smithi031 ceph-mon[24497]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:26 smithi018 ceph-mon[23740]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:26 smithi018 ceph-mon[28324]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:28 smithi031 ceph-mon[24497]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:29.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:29.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:28 smithi018 ceph-mon[28324]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:29.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:29.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:28 smithi018 ceph-mon[23740]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:29.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:30.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:30.855 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:30.855 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:30 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:30.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T00:05:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:31.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:31.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:30 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:32 smithi018 ceph-mon[23740]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:32 smithi018 ceph-mon[28324]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:32 smithi031 ceph-mon[24497]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:33.350 INFO:teuthology.orchestra.run.smithi018.stdout:Scheduled to redeploy mgr.y on host 'smithi018' 2024-09-21T00:05:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:33] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-21T00:05:33.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:33.503Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/188519067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:33.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/188519067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:33.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:33.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:33.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:05:33.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:33 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:33.982 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps --refresh' 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/188519067' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:05:34.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:05:34.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:34.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:05:34.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:05:34.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:33 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:05:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[23740]: from='client.25075 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[23740]: Schedule redeploy daemon mgr.y 2024-09-21T00:05:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[23740]: Deploying daemon mgr.y on smithi018 2024-09-21T00:05:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[23740]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[28324]: from='client.25075 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[28324]: Schedule redeploy daemon mgr.y 2024-09-21T00:05:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[28324]: Deploying daemon mgr.y on smithi018 2024-09-21T00:05:34.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:34 smithi018 ceph-mon[28324]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:34 smithi031 ceph-mon[24497]: from='client.25075 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:34 smithi031 ceph-mon[24497]: Schedule redeploy daemon mgr.y 2024-09-21T00:05:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:34 smithi031 ceph-mon[24497]: Deploying daemon mgr.y on smithi018 2024-09-21T00:05:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:34 smithi031 ceph-mon[24497]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:35.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:35.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:36.295 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:05:36.295 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (2m) 6s ago 10m 24.3M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:05:36.295 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (2m) 6s ago 10m 44.3M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (2m) 6s ago 9m 51.0M - 3.5 e1d6a67b021e 32129445ad31 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (5m) 6s ago 15m 504M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:9283 running (16m) 6s ago 16m 410M - 17.2.0 e1d6a67b021e 768baae6a4fb 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (17m) 6s ago 17m 56.8M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (15m) 6s ago 15m 47.2M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (15m) 6s ago 15m 47.6M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (2m) 6s ago 11m 17.3M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (2m) 6s ago 11m 14.9M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (15m) 6s ago 15m 53.1M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (14m) 6s ago 14m 52.7M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (14m) 6s ago 14m 51.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (13m) 6s ago 13m 51.0M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:05:36.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (13m) 6s ago 13m 52.7M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (13m) 6s ago 13m 49.6M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (12m) 6s ago 12m 50.9M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (12m) 6s ago 12m 53.6M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (2m) 6s ago 11m 61.0M - 2.43.0 a07b618ecd1d 76853f686e1e 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (10m) 6s ago 10m 74.9M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:05:36.297 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (10m) 6s ago 10m 75.9M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:05:36.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:36 smithi018 ceph-mon[23740]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:36.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:36 smithi018 ceph-mon[28324]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:37.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-21T00:05:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:36 smithi031 ceph-mon[24497]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:38.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:37 smithi018 ceph-mon[23740]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:38.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:37 smithi018 ceph-mon[28324]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:38.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:37 smithi031 ceph-mon[24497]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:05:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:38 smithi031 ceph-mon[24497]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:39.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:38 smithi018 ceph-mon[23740]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:38 smithi018 ceph-mon[28324]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:41.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:40 smithi018 ceph-mon[23740]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:41.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:40 smithi018 ceph-mon[28324]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:40 smithi031 ceph-mon[24497]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:43.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:42 smithi031 ceph-mon[24497]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-21T00:05:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:42 smithi018 ceph-mon[23740]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-21T00:05:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:42 smithi018 ceph-mon[28324]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-21T00:05:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:43] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-21T00:05:43.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:45.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:44 smithi018 ceph-mon[23740]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:44 smithi018 ceph-mon[28324]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:44 smithi031 ceph-mon[24497]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-21T00:05:46.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:45 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:45 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:45 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:46 smithi018 ceph-mon[23740]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:46 smithi018 ceph-mon[28324]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:47.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:46 smithi031 ceph-mon[24497]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:48 smithi031 ceph-mon[24497]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:49.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:48 smithi018 ceph-mon[23740]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:48 smithi018 ceph-mon[28324]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:50 smithi018 ceph-mon[23740]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:50 smithi018 ceph-mon[28324]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:50 smithi031 ceph-mon[24497]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-09-21T00:05:53.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:52 smithi018 ceph-mon[23740]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:52 smithi018 ceph-mon[28324]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:52 smithi031 ceph-mon[24497]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:05:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:05:53] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-21T00:05:53.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:54 smithi018 ceph-mon[23740]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:54 smithi018 ceph-mon[28324]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:54 smithi031 ceph-mon[24497]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:05:56.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:55 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:55 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:55 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:05:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:56 smithi018 ceph-mon[23740]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:56 smithi018 ceph-mon[28324]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:56 smithi031 ceph-mon[24497]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:58 smithi031 ceph-mon[24497]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:05:58 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:59.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:05:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:05:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:05:59.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:58 smithi018 ceph-mon[23740]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:05:58 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:05:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:58 smithi018 ceph-mon[28324]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:05:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:05:58 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:01.067 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:00 smithi018 ceph-mon[23740]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:01.067 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:00 smithi018 ceph-mon[28324]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:00 smithi031 ceph-mon[24497]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:01.367 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:01 smithi018 systemd[1]: Stopping Ceph mgr.y for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:06:01.670 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:01 smithi018 podman[61273]: 2024-09-21 00:06:01.366171957 +0000 UTC m=+0.109494411 container died 768baae6a4fba87846662400a6920b5a79e2be877c91a026a2a2470c667a8758 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_CLEAN=True, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-type=git, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T00:06:01.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:01 smithi018 podman[61273]: 2024-09-21 00:06:01.760182675 +0000 UTC m=+0.503505121 container cleanup 768baae6a4fba87846662400a6920b5a79e2be877c91a026a2a2470c667a8758 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-09-21T00:06:01.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:01 smithi018 bash[61273]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y 2024-09-21T00:06:02.358 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 podman[61287]: 2024-09-21 00:06:02.012022008 +0000 UTC m=+0.647574665 container remove 768baae6a4fba87846662400a6920b5a79e2be877c91a026a2a2470c667a8758 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8) 2024-09-21T00:06:02.358 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:06:02.358 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T00:06:02.359 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 systemd[1]: Stopped Ceph mgr.y for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:06:02.359 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service: Consumed 1min 4.407s CPU time. 2024-09-21T00:06:02.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 systemd[1]: Starting Ceph mgr.y for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:06:02.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 podman[61412]: 2024-09-21 00:06:02.472223691 +0000 UTC m=+0.016533381 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:06:03.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:02 smithi018 ceph-mon[23740]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:03.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:02 smithi018 ceph-mon[28324]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:03.042 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 podman[61412]: 2024-09-21 00:06:02.712905762 +0000 UTC m=+0.257215448 container create e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:06:03.042 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 podman[61412]: 2024-09-21 00:06:02.932031363 +0000 UTC m=+0.476341044 container init e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:06:03.042 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 podman[61412]: 2024-09-21 00:06:02.935253464 +0000 UTC m=+0.479563152 container start e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T00:06:03.042 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:02 smithi018 ceph-mgr[61431]: -- 172.21.15.18:0/3489280659 <== mon.1 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5572ef1a90e0 con 0x5572ef187000 2024-09-21T00:06:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:02 smithi031 ceph-mon[24497]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:03.361 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 bash[61412]: e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 2024-09-21T00:06:03.361 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 systemd[1]: Started Ceph mgr.y for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:06:03.361 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.062+0000 7f3c3a994040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:06:03.361 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.127+0000 7f3c3a994040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:06:03.361 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.288+0000 7f3c3a994040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:06:03.362 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.360+0000 7f3c3a994040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:06:03.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.531+0000 7f3c3a994040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:06:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:03] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-21T00:06:03.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:04.033 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:03.685+0000 7f3c3a994040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:06:04.034 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:04.030+0000 7f3c3a994040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:06:04.311 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:04.094+0000 7f3c3a994040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:06:04.311 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:04.309+0000 7f3c3a994040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:06:04.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[23740]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:04 smithi018 ceph-mon[28324]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:04 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:04 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:04 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:04 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:04.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:04 smithi031 ceph-mon[24497]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:05.419 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:05 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:05.420 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:05 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:05.420 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:05 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:05.345+0000 7f3c3a994040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:06:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:05 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:05.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:05 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:05.418+0000 7f3c3a994040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:06:06.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.231+0000 7f3c3a994040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:06:06.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.354+0000 7f3c3a994040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:06:06.822 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-mon[23740]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:06.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-mon[28324]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:06.822 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.506+0000 7f3c3a994040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:06:06.822 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.570+0000 7f3c3a994040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:06:06.823 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.747+0000 7f3c3a994040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:06:06.823 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.820+0000 7f3c3a994040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:06:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:06 smithi031 ceph-mon[24497]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:07.078 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:06.884+0000 7f3c3a994040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:06:07.078 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.055+0000 7f3c3a994040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:06:07.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.125+0000 7f3c3a994040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:06:07.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.198+0000 7f3c3a994040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:06:07.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.286+0000 7f3c3a994040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:06:07.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.349+0000 7f3c3a994040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:06:07.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:07.423+0000 7f3c3a994040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:06:08.660 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.660 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.660 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: Standby manager daemon y restarted 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: Standby manager daemon y started 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:06:08.661 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: Standby manager daemon y restarted 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: Standby manager daemon y started 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:06:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: Standby manager daemon y restarted 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: Standby manager daemon y started 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:08.266+0000 7f3c3a994040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:06:08.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:06:08] ENGINE Bus STARTING 2024-09-21T00:06:08.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: CherryPy Checker: 2024-09-21T00:06:08.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: The Application mounted at '' has an empty config. 2024-09-21T00:06:08.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:06:08.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:06:08] ENGINE Serving on http://:::9283 2024-09-21T00:06:08.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:06:08 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:06:08] ENGINE Bus STARTED 2024-09-21T00:06:09.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:08 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:09 smithi018 ceph-mon[23740]: mgrmap e26: x(active, since 3m), standbys: y 2024-09-21T00:06:09.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:09 smithi018 ceph-mon[28324]: mgrmap e26: x(active, since 3m), standbys: y 2024-09-21T00:06:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:09 smithi031 ceph-mon[24497]: mgrmap e26: x(active, since 3m), standbys: y 2024-09-21T00:06:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:10 smithi018 ceph-mon[23740]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:10.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:10 smithi018 ceph-mon[28324]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:10 smithi031 ceph-mon[24497]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:06:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:06:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:12 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:06:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:12 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:06:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:13] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:06:13.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:13 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:13 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:13 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:14 smithi018 ceph-mon[23740]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:14 smithi018 ceph-mon[28324]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:14 smithi031 ceph-mon[24497]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:15 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:15 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:15 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:16 smithi018 ceph-mon[23740]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:16 smithi018 ceph-mon[28324]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:16 smithi031 ceph-mon[24497]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:18 smithi031 ceph-mon[24497]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:19.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:18 smithi018 ceph-mon[28324]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:18 smithi018 ceph-mon[23740]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:20 smithi018 ceph-mon[23740]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:20 smithi018 ceph-mon[28324]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:20 smithi031 ceph-mon[24497]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:22 smithi018 ceph-mon[23740]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:22 smithi018 ceph-mon[28324]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:22 smithi031 ceph-mon[24497]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:23] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:06:23.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:24 smithi018 ceph-mon[23740]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:24 smithi018 ceph-mon[28324]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:24 smithi031 ceph-mon[24497]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:26 smithi018 ceph-mon[23740]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:26 smithi018 ceph-mon[28324]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:26 smithi031 ceph-mon[24497]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:28 smithi031 ceph-mon[24497]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:29.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:28 smithi018 ceph-mon[23740]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:28 smithi018 ceph-mon[28324]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:30 smithi018 ceph-mon[23740]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:30 smithi018 ceph-mon[28324]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:30 smithi031 ceph-mon[24497]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:32 smithi018 ceph-mon[23740]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:32 smithi018 ceph-mon[28324]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:32 smithi031 ceph-mon[24497]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:33] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:06:33.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:34 smithi018 ceph-mon[28324]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:34 smithi018 ceph-mon[23740]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:34 smithi031 ceph-mon[24497]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:36 smithi018 ceph-mon[23740]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:36 smithi018 ceph-mon[28324]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:36 smithi031 ceph-mon[24497]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:38 smithi031 ceph-mon[24497]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:39.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:38 smithi018 ceph-mon[23740]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:38 smithi018 ceph-mon[28324]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:40 smithi018 ceph-mon[23740]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:40 smithi018 ceph-mon[28324]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:40 smithi031 ceph-mon[24497]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:42 smithi018 ceph-mon[23740]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:42 smithi018 ceph-mon[28324]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:42 smithi031 ceph-mon[24497]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:43] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:06:43.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:44 smithi018 ceph-mon[23740]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:44 smithi018 ceph-mon[28324]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:44 smithi031 ceph-mon[24497]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:45.865 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:45 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:45 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:45 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:46 smithi018 ceph-mon[23740]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:46 smithi018 ceph-mon[28324]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:46 smithi031 ceph-mon[24497]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:48 smithi031 ceph-mon[24497]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:49.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:48 smithi018 ceph-mon[23740]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:48 smithi018 ceph-mon[28324]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:50 smithi018 ceph-mon[23740]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:50 smithi018 ceph-mon[28324]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:50 smithi031 ceph-mon[24497]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:52 smithi018 ceph-mon[23740]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:52 smithi018 ceph-mon[28324]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:52 smithi031 ceph-mon[24497]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:06:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:06:53] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:06:53.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:54 smithi018 ceph-mon[23740]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:54 smithi018 ceph-mon[28324]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:54 smithi031 ceph-mon[24497]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:06:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:55 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:55 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:55 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:06:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:56 smithi018 ceph-mon[23740]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:56 smithi018 ceph-mon[28324]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:56 smithi031 ceph-mon[24497]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:58 smithi031 ceph-mon[24497]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:06:58 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:59.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:06:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:06:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:06:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:58 smithi018 ceph-mon[23740]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:06:58 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:06:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:58 smithi018 ceph-mon[28324]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:06:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:06:58 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:00 smithi018 ceph-mon[23740]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:00 smithi018 ceph-mon[28324]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:00 smithi031 ceph-mon[24497]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:02 smithi018 ceph-mon[23740]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:02 smithi018 ceph-mon[28324]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:02 smithi031 ceph-mon[24497]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:03] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:07:03.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:04 smithi018 ceph-mon[23740]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:04 smithi018 ceph-mon[28324]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:04 smithi031 ceph-mon[24497]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:05 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:05 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:05 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:06 smithi018 ceph-mon[23740]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:06 smithi018 ceph-mon[28324]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:06 smithi031 ceph-mon[24497]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:08 smithi031 ceph-mon[24497]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:09.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:08 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:08 smithi018 ceph-mon[23740]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:08 smithi018 ceph-mon[28324]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:10 smithi018 ceph-mon[23740]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:10 smithi018 ceph-mon[28324]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:10 smithi031 ceph-mon[24497]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:12 smithi018 ceph-mon[23740]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:12 smithi018 ceph-mon[28324]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:12 smithi031 ceph-mon[24497]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:13] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:07:13.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:13.503Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:14.162 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:13 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:14.162 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:13 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:13 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:14 smithi018 ceph-mon[23740]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:14 smithi018 ceph-mon[28324]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:14 smithi031 ceph-mon[24497]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:16.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:15 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:16.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:15 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:15 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:16 smithi018 ceph-mon[23740]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:16 smithi018 ceph-mon[28324]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:16 smithi031 ceph-mon[24497]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:07:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:07:18.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:18 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:19.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:07:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:07:19.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:18 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:07:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:20 smithi018 ceph-mon[23740]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:20 smithi018 ceph-mon[28324]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:20 smithi031 ceph-mon[24497]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:22 smithi018 ceph-mon[23740]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:22 smithi018 ceph-mon[28324]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:22 smithi031 ceph-mon[24497]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:23] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-21T00:07:23.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:24 smithi018 ceph-mon[23740]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:24 smithi018 ceph-mon[28324]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:24 smithi031 ceph-mon[24497]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:26 smithi018 ceph-mon[23740]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:26 smithi018 ceph-mon[28324]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:26 smithi031 ceph-mon[24497]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:28 smithi031 ceph-mon[24497]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:29.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:29.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:28 smithi018 ceph-mon[23740]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:28 smithi018 ceph-mon[28324]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:30 smithi018 ceph-mon[23740]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:30 smithi018 ceph-mon[28324]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:30 smithi031 ceph-mon[24497]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:32 smithi018 ceph-mon[23740]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:32 smithi018 ceph-mon[28324]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:32 smithi031 ceph-mon[24497]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:33] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.43.0" 2024-09-21T00:07:33.752 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:34 smithi018 ceph-mon[23740]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:34 smithi018 ceph-mon[28324]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:34 smithi031 ceph-mon[24497]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:36 smithi018 ceph-mon[23740]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:36 smithi018 ceph-mon[28324]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:36 smithi031 ceph-mon[24497]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:39.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:38 smithi031 ceph-mon[24497]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:39.001 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:38 smithi018 ceph-mon[23740]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:38 smithi018 ceph-mon[28324]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:40 smithi018 ceph-mon[23740]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:40 smithi018 ceph-mon[28324]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:40 smithi031 ceph-mon[24497]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:42 smithi018 ceph-mon[23740]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:42 smithi018 ceph-mon[28324]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:42 smithi031 ceph-mon[24497]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:07:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:43] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-21T00:07:43.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:44 smithi018 ceph-mon[23740]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:44 smithi018 ceph-mon[28324]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:44 smithi031 ceph-mon[24497]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:46.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:45 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:45 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:45 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:46 smithi018 ceph-mon[28324]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:46 smithi018 ceph-mon[23740]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:46 smithi031 ceph-mon[24497]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:49.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:48 smithi031 ceph-mon[24497]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:49.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:48 smithi018 ceph-mon[23740]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:48 smithi018 ceph-mon[28324]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:50 smithi018 ceph-mon[23740]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:50 smithi018 ceph-mon[28324]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:50 smithi031 ceph-mon[24497]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:52 smithi018 ceph-mon[23740]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:52 smithi018 ceph-mon[28324]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:52 smithi031 ceph-mon[24497]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:07:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:07:53] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-21T00:07:53.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi018\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", domainname=\"(none)\", instance=\"smithi018\", job=\"node\", machine=\"x86_64\", nodename=\"smithi018\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:54 smithi018 ceph-mon[28324]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:54 smithi018 ceph-mon[23740]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:54 smithi031 ceph-mon[24497]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:07:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:55 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:55 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:55 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:07:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:56 smithi018 ceph-mon[23740]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:56 smithi018 ceph-mon[28324]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:56 smithi031 ceph-mon[24497]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:58 smithi031 ceph-mon[24497]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:07:58 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:59.002 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:07:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:07:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"172.21.15.31:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"c1f9e122-77aa-11ef-baf5-efdc52797490\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T00:07:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:58 smithi018 ceph-mon[28324]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:07:58 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:07:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:58 smithi018 ceph-mon[23740]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:07:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:07:58 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:00 smithi018 ceph-mon[23740]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:00 smithi018 ceph-mon[28324]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:00 smithi031 ceph-mon[24497]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:02 smithi018 ceph-mon[28324]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:02 smithi018 ceph-mon[23740]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:03.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:02 smithi031 ceph-mon[24497]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:03.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:03] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.43.0" 2024-09-21T00:08:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:04 smithi018 ceph-mon[28324]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:04 smithi018 ceph-mon[23740]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:04 smithi031 ceph-mon[24497]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:05 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:05 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:05 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:06 smithi018 ceph-mon[23740]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:06 smithi018 ceph-mon[28324]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:06 smithi031 ceph-mon[24497]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:08 smithi018 ceph-mon[23740]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:08 smithi018 ceph-mon[28324]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:08 smithi031 ceph-mon[24497]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:10 smithi018 ceph-mon[23740]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:10 smithi018 ceph-mon[28324]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:10 smithi031 ceph-mon[24497]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:12 smithi018 ceph-mon[23740]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:12 smithi018 ceph-mon[28324]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:12 smithi031 ceph-mon[24497]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:13] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.43.0" 2024-09-21T00:08:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:13 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:13 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:13 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:14 smithi018 ceph-mon[23740]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:14 smithi018 ceph-mon[28324]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:14 smithi031 ceph-mon[24497]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:15 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:16.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:15 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:15 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:16 smithi018 ceph-mon[23740]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:16 smithi018 ceph-mon[28324]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:16 smithi031 ceph-mon[24497]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[23740]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[23740]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[28324]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:08:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:18 smithi018 ceph-mon[28324]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:08:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:18 smithi031 ceph-mon[24497]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:18 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:08:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:18 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:08:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:18 smithi031 ceph-mon[24497]: from='mgr.24794 ' entity='mgr.x' 2024-09-21T00:08:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:20 smithi018 ceph-mon[23740]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:20 smithi018 ceph-mon[28324]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:20 smithi031 ceph-mon[24497]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:22 smithi031 ceph-mon[24497]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:22 smithi018 ceph-mon[23740]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:23.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:22 smithi018 ceph-mon[28324]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:23] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.43.0" 2024-09-21T00:08:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:24 smithi031 ceph-mon[24497]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:25.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:24 smithi018 ceph-mon[23740]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:25.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:24 smithi018 ceph-mon[28324]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:25 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:25 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:25 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:26 smithi031 ceph-mon[24497]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:27.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:26 smithi018 ceph-mon[23740]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:27.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:26 smithi018 ceph-mon[28324]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:28 smithi031 ceph-mon[24497]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:28 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:28 smithi018 ceph-mon[23740]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:28 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:28 smithi018 ceph-mon[28324]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:28 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:30 smithi031 ceph-mon[24497]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:30 smithi018 ceph-mon[23740]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:30 smithi018 ceph-mon[28324]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:32 smithi031 ceph-mon[24497]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:33.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:32 smithi018 ceph-mon[23740]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:32 smithi018 ceph-mon[28324]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:33] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:08:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:34 smithi031 ceph-mon[24497]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:35.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:34 smithi018 ceph-mon[23740]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:34 smithi018 ceph-mon[28324]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:35 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:35 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:35 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:36 smithi031 ceph-mon[24497]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:36 smithi018 ceph-mon[23740]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:36 smithi018 ceph-mon[28324]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:38 smithi031 ceph-mon[24497]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:38 smithi018 ceph-mon[23740]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:38 smithi018 ceph-mon[28324]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:40.829 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:08:41.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:40 smithi018 ceph-mon[23740]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:41.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:40 smithi018 ceph-mon[28324]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:40 smithi031 ceph-mon[24497]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:43.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:42 smithi018 ceph-mon[23740]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:43.040 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:42 smithi018 ceph-mon[28324]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (5m) 85s ago 13m 24.3M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (5m) 2m ago 13m 44.2M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (5m) 85s ago 12m 53.6M - 3.5 e1d6a67b021e 32129445ad31 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (8m) 2m ago 18m 506M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (2m) 85s ago 20m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (20m) 85s ago 20m 59.3M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:08:43.040 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (18m) 2m ago 18m 41.2M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (18m) 85s ago 18m 45.8M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (6m) 85s ago 14m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (5m) 2m ago 14m 16.8M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (18m) 85s ago 18m 53.8M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (17m) 85s ago 17m 53.7M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (17m) 85s ago 17m 51.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (16m) 85s ago 16m 52.4M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (16m) 2m ago 16m 53.5M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (16m) 2m ago 16m 50.3M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (15m) 2m ago 15m 51.9M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (15m) 2m ago 15m 54.8M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (5m) 2m ago 14m 61.0M - 2.43.0 a07b618ecd1d 76853f686e1e 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (13m) 85s ago 13m 75.6M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:08:43.041 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (13m) 2m ago 13m 76.0M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:08:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:42 smithi031 ceph-mon[24497]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:43.655 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:08:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:43] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:08:44.226 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:43 smithi018 ceph-mon[23740]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:08:44.226 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:43 smithi018 ceph-mon[23740]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:44.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:43 smithi018 ceph-mon[28324]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:08:44.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:43 smithi018 ceph-mon[28324]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:43 smithi031 ceph-mon[24497]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:08:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:43 smithi031 ceph-mon[24497]: from='mgr.24794 172.21.15.31:0/512815587' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:45.150 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:44 smithi018 ceph-mon[28324]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:45.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:44 smithi018 ceph-mon[23740]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:44 smithi031 ceph-mon[24497]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:45.682 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:08:45.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:08:45.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:08:45.684 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:08:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:45 smithi031 ceph-mon[24497]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/341651137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:08:46.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:45 smithi018 ceph-mon[28324]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:46.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/341651137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:08:46.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:45 smithi018 ceph-mon[23740]: from='client.15111 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:08:46.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/341651137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:08:46.347 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:08:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:46 smithi031 ceph-mon[24497]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:46 smithi018 ceph-mon[23740]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:46 smithi018 ceph-mon[28324]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:48.398 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T00:08:48.983 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:48 smithi018 ceph-mon[28324]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:48.983 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:48 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2917922019' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:08:48.983 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:48 smithi018 ceph-mon[23740]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:48.983 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:48 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2917922019' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:08:49.008 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph -s' 2024-09-21T00:08:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:48 smithi031 ceph-mon[24497]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:48 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2917922019' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: cluster: 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: id: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: health: HEALTH_OK 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: services: 2024-09-21T00:08:51.143 INFO:teuthology.orchestra.run.smithi018.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: mgr: x(active, since 6m), standbys: y 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: data: 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: pools: 6 pools, 161 pgs 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: objects: 241 objects, 585 KiB 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: usage: 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: pgs: 161 active+clean 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: io: 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-09-21T00:08:51.144 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:08:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:50 smithi031 ceph-mon[24497]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:50 smithi018 ceph-mon[23740]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:50 smithi018 ceph-mon[28324]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:51.833 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph mgr fail' 2024-09-21T00:08:52.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:51 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2161718967' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:08:52.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:51 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2161718967' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:08:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:51 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2161718967' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:08:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:52 smithi031 ceph-mon[24497]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:53.306 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:52 smithi018 ceph-mon[23740]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:53.307 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:52 smithi018 ceph-mon[28324]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:08:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:08:53] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-21T00:08:55.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[28324]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3243388327' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[28324]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE Bus STOPPING 2024-09-21T00:08:55.250 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE Bus STOPPED 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE Bus STARTING 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[23740]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3243388327' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[23740]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T00:08:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:54 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-mon[24497]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3243388327' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-mon[24497]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:54.991+0000 7f3f2d89d640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ignoring --setuser ceph since I am not root 2024-09-21T00:08:55.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ignoring --setgroup ceph since I am not root 2024-09-21T00:08:55.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mgr[44199]: -- 172.21.15.31:0/4174286100 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c5a6def0e0 con 0x55c5a6dcd000 2024-09-21T00:08:55.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.148+0000 7f8f10b18040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:08:55.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.209+0000 7f8f10b18040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:08:55.645 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE Serving on http://:::9283 2024-09-21T00:08:55.645 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:08:55] ENGINE Bus STARTED 2024-09-21T00:08:55.670 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-21T00:08:55.743 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.363+0000 7f8f10b18040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:08:55.743 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.433+0000 7f8f10b18040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:08:55.743 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.590+0000 7f8f10b18040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:08:55.743 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:55.741+0000 7f8f10b18040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:08:56.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:08:56.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: mgrmap e27: y(active, starting, since 0.99336s), standbys: x 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:08:56.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:08:56.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: Manager daemon y is now available 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:55 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: mgrmap e27: y(active, starting, since 0.99336s), standbys: x 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:08:56.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: Manager daemon y is now available 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:08:56.316 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: mgrmap e27: y(active, starting, since 0.99336s), standbys: x 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:08:56.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:08:56.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.319 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:55 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:08:56.365 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:56.100+0000 7f8f10b18040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:08:56.365 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:56.164+0000 7f8f10b18040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:08:56.365 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:56.364+0000 7f8f10b18040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:08:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:56 smithi031 ceph-mon[24497]: mgrmap e28: y(active, since 1.99905s), standbys: x 2024-09-21T00:08:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:56 smithi031 ceph-mon[24497]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:08:57.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:56 smithi018 ceph-mon[23740]: mgrmap e28: y(active, since 1.99905s), standbys: x 2024-09-21T00:08:57.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:56 smithi018 ceph-mon[23740]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:08:57.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:56 smithi018 ceph-mon[28324]: mgrmap e28: y(active, since 1.99905s), standbys: x 2024-09-21T00:08:57.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:56 smithi018 ceph-mon[28324]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:08:57.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:57.338+0000 7f8f10b18040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:08:57.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:57.415+0000 7f8f10b18040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: mgrmap e29: y(active, since 3s), standbys: x 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: [21/Sep/2024:00:08:57] ENGINE Bus STARTING 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: [21/Sep/2024:00:08:57] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: [21/Sep/2024:00:08:57] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: [21/Sep/2024:00:08:57] ENGINE Bus STARTED 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[28324]: [21/Sep/2024:00:08:57] ENGINE Error in HTTPServer.serve 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:08:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.do_handshake() 2024-09-21T00:08:58.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: mgrmap e29: y(active, since 3s), standbys: x 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: [21/Sep/2024:00:08:57] ENGINE Bus STARTING 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: [21/Sep/2024:00:08:57] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: [21/Sep/2024:00:08:57] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: [21/Sep/2024:00:08:57] ENGINE Bus STARTED 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:08:58 smithi018 ceph-mon[23740]: [21/Sep/2024:00:08:57] ENGINE Error in HTTPServer.serve 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:08:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.do_handshake() 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:08:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:08:57.914+0000 7f3b8d93b640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:00:08:57] ENGINE Error in HTTPServer.serve 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._connections.run(self.expiration_interval) 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._run(expiration_interval) 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:08:58.179 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s = self.context.wrap_socket( 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.sslsocket_class._create( 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self.do_handshake() 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._sslobj.do_handshake() 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:08:58.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:08:57 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:08:58.229 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: mgrmap e29: y(active, since 3s), standbys: x 2024-09-21T00:08:58.229 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: [21/Sep/2024:00:08:57] ENGINE Bus STARTING 2024-09-21T00:08:58.229 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: [21/Sep/2024:00:08:57] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:08:58.229 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: [21/Sep/2024:00:08:57] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:08:58.229 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: [21/Sep/2024:00:08:57] ENGINE Bus STARTED 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-mon[24497]: [21/Sep/2024:00:08:57] ENGINE Error in HTTPServer.serve 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._run(expiration_interval) 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:08:58.230 INFO:journalctl@ceph.mon.b.smithi031.stdout: s = self.context.wrap_socket( 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.sslsocket_class._create( 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.do_handshake() 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._sslobj.do_handshake() 2024-09-21T00:08:58.231 INFO:journalctl@ceph.mon.b.smithi031.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:08:58.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.228+0000 7f8f10b18040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:08:58.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.353+0000 7f8f10b18040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:08:58.803 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.506+0000 7f8f10b18040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:08:58.804 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.571+0000 7f8f10b18040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:08:58.804 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.730+0000 7f8f10b18040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:08:58.804 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.802+0000 7f8f10b18040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:08:59.119 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:58 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:58.865+0000 7f8f10b18040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:08:59.119 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.049+0000 7f8f10b18040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:08:59.119 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.117+0000 7f8f10b18040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:08:59.407 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.187+0000 7f8f10b18040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:08:59.407 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.277+0000 7f8f10b18040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:08:59.407 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.340+0000 7f8f10b18040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:08:59.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:08:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:08:59.405+0000 7f8f10b18040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:09:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[28324]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[28324]: mgrmap e30: y(active, since 5s), standbys: x 2024-09-21T00:09:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[23740]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:00.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[23740]: mgrmap e30: y(active, since 5s), standbys: x 2024-09-21T00:09:00.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:00 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-mon[24497]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:00.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-mon[24497]: mgrmap e30: y(active, since 5s), standbys: x 2024-09-21T00:09:00.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:00.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:09:00.268+0000 7f8f10b18040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:09:00.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:09:00] ENGINE Bus STARTING 2024-09-21T00:09:00.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: CherryPy Checker: 2024-09-21T00:09:00.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: The Application mounted at '' has an empty config. 2024-09-21T00:09:00.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:09:00.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:09:00] ENGINE Serving on http://:::9283 2024-09-21T00:09:00.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:09:00] ENGINE Bus STARTED 2024-09-21T00:09:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: Standby manager daemon x restarted 2024-09-21T00:09:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-21T00:09:01.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:09:01.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:01 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:09:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: Standby manager daemon x restarted 2024-09-21T00:09:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-21T00:09:01.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:09:01.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:09:01.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:09:01.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:01 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[28324]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[28324]: mgrmap e31: y(active, since 7s), standbys: x 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[23740]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:09:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[23740]: mgrmap e31: y(active, since 7s), standbys: x 2024-09-21T00:09:02.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:02.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:02 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:02.467 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:02 smithi031 ceph-mon[24497]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:09:02.467 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:02 smithi031 ceph-mon[24497]: mgrmap e31: y(active, since 7s), standbys: x 2024-09-21T00:09:02.467 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:02 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:02.467 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:02 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.460 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:09:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.461 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.462 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:03 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:03.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:09:03.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:03.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:03 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:09:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:04.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:04 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:09:04.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:04.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:04.530 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:04.530 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:04 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:06.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:09:06.058+0000 7f3bcabf2640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:06.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Creating ceph-iscsi config... 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return _run_code(code, main_globals, None, 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: exec(code, run_globals) 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:06.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:09:06.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.wait_async( 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return future.result(timeout) 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.__get_result() 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: raise self._exception 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:09:06.428 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: out, err, code = await self._run_cephadm( 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: raise OrchestratorError( 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Creating ceph-iscsi config... 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return _run_code(code, main_globals, None, 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:06.429 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: exec(code, run_globals) 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:06.430 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:06.698 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:06.287Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.31:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.31:8765: connect: connection refused" 2024-09-21T00:09:06.698 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:06.290Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.31:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.31:8765: connect: connection refused" 2024-09-21T00:09:06.698 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:06.290Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.31:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.31:8765: connect: connection refused" 2024-09-21T00:09:06.698 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:06.290Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.31:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.31:8765: connect: connection refused" 2024-09-21T00:09:06.698 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:06.290Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.31:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.31:8765: connect: connection refused" 2024-09-21T00:09:06.991 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:06 smithi031 ceph-mon[24497]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:09:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-mon[23740]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:09:07.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:06 smithi018 ceph-mon[28324]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:09:07.910 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:07 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:07.910 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:07.911 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:09:07.912 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:09:07.913 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:07.914 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:07.915 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:07 smithi031 ceph-mon[24497]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:09:07.916 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:07 smithi031 ceph-mon[24497]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:09:08.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:08.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:09:08.179 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:08.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:08.181 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[23740]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[23740]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:09:08.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:09:08.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[28324]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:09:08.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:07 smithi018 ceph-mon[28324]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:09:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:08 smithi031 ceph-mon[24497]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:09:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:08 smithi018 ceph-mon[23740]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:09:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:08 smithi018 ceph-mon[28324]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:09:09.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 systemd[1]: Stopping Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:09:09.781 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.503Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:09:09.781 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.503Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T00:09:09.781 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.504Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.505Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.505Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.505Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[48184]: ts=2024-09-21T00:09:09.505Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T00:09:09.782 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 podman[53527]: 2024-09-21 00:09:09.514013111 +0000 UTC m=+0.111821263 container died 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.155 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 podman[53527]: 2024-09-21 00:09:09.878680181 +0000 UTC m=+0.476488332 container cleanup 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.155 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:09 smithi031 bash[53527]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-21T00:09:10.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 podman[53539]: 2024-09-21 00:09:10.154537188 +0000 UTC m=+0.637870869 container remove 76853f686e1edd3298c764603c2aa8b82220f5f9febd797f66e5b0617ea5f713 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T00:09:10.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 systemd[1]: Stopped Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:09:10.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Consumed 2.454s CPU time. 2024-09-21T00:09:10.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 systemd[1]: Starting Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:09:10.789 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-mon[24497]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:10.789 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:10.789 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 podman[53634]: 2024-09-21 00:09:10.436032152 +0000 UTC m=+0.017936957 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 podman[53634]: 2024-09-21 00:09:10.547255848 +0000 UTC m=+0.129160644 container create 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 podman[53634]: 2024-09-21 00:09:10.724154422 +0000 UTC m=+0.306059228 container init 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 podman[53634]: 2024-09-21 00:09:10.726896426 +0000 UTC m=+0.308801223 container start 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.750Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.750Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T00:09:10.790 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.750Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi031 (none))" 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.750Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.750Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.752Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.752Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.754Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.754Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.756Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T00:09:10.791 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.757Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.441235ms 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.757Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.765Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.781Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.788Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.788Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T00:09:10.792 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.788Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.776µs wal_replay_duration=30.712916ms wbl_replay_duration=131ns total_replay_duration=32.208185ms 2024-09-21T00:09:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:10 smithi018 ceph-mon[23740]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:10 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:10 smithi018 ceph-mon[28324]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:10 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:11.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.792Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T00:09:11.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.792Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.792Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 bash[53634]: 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.816Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.681216ms db_storage=751ns remote_storage=1.21µs web_handler=377ns query_engine=655ns scrape=573.429µs scrape_sd=128.899µs notify=13.306µs notify_sd=29.72µs rules=23.584015ms tracing=6.42µs 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.816Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:09:10.816Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T00:09:11.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:09:10 smithi031 systemd[1]: Started Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:12.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:11 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:11 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:11 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:11 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:11 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:09:13.125 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:12 smithi031 ceph-mon[24497]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:12 smithi018 ceph-mon[23740]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:12 smithi018 ceph-mon[28324]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:14 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:09:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:14 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:16.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:15 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:16.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:15 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:16.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:15 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:15 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:15 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:09:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:15 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:09:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:16 smithi018 ceph-mon[23740]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:16 smithi018 ceph-mon[28324]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:16 smithi031 ceph-mon[24497]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/563589315' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:09:18.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:18 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/563589315' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-21T00:09:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:18 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/563589315' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:09:20.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:20 smithi018 ceph-mon[28324]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:20.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:20 smithi018 ceph-mon[23740]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:20 smithi031 ceph-mon[24497]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:09:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:09:21] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:09:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:09:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:09:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:24 smithi018 ceph-mon[23740]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:24 smithi018 ceph-mon[28324]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:24 smithi031 ceph-mon[24497]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:25 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:25 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:25 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:26 smithi018 ceph-mon[23740]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:26 smithi018 ceph-mon[28324]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:26 smithi031 ceph-mon[24497]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T00:09:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:28 smithi018 ceph-mon[28324]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:28 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:28 smithi018 ceph-mon[23740]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:28 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:28 smithi031 ceph-mon[24497]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:28 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:31.053 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:30 smithi031 ceph-mon[24497]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:30 smithi018 ceph-mon[28324]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:30 smithi018 ceph-mon[23740]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:09:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:09:31] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-21T00:09:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:32 smithi018 ceph-mon[23740]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:33.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:32 smithi018 ceph-mon[28324]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:33.263 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:32 smithi031 ceph-mon[24497]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:34 smithi018 ceph-mon[23740]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:34 smithi018 ceph-mon[28324]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:34 smithi031 ceph-mon[24497]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:36 smithi018 ceph-mon[23740]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:36 smithi018 ceph-mon[28324]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:36 smithi031 ceph-mon[24497]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:38 smithi018 ceph-mon[28324]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:38 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:38 smithi018 ceph-mon[23740]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:38 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:38 smithi031 ceph-mon[24497]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:38 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:40 smithi018 ceph-mon[28324]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:40 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:40 smithi018 ceph-mon[23740]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:40 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:40 smithi031 ceph-mon[24497]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:40 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:09:41] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-21T00:09:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:42 smithi018 ceph-mon[23740]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:42 smithi018 ceph-mon[28324]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:42 smithi031 ceph-mon[24497]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:44 smithi018 ceph-mon[23740]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:44 smithi018 ceph-mon[28324]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:44 smithi031 ceph-mon[24497]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:46 smithi018 ceph-mon[28324]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:46 smithi018 ceph-mon[23740]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:46 smithi031 ceph-mon[24497]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:48 smithi018 ceph-mon[23740]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:48 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:48 smithi018 ceph-mon[28324]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:48 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:48 smithi031 ceph-mon[24497]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:48 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:50 smithi018 ceph-mon[23740]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:50 smithi018 ceph-mon[28324]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:50 smithi031 ceph-mon[24497]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:09:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:09:51] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-21T00:09:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:52 smithi018 ceph-mon[28324]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:52 smithi018 ceph-mon[23740]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:52 smithi031 ceph-mon[24497]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:54 smithi018 ceph-mon[23740]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:54 smithi018 ceph-mon[28324]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:54 smithi031 ceph-mon[24497]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:09:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:56 smithi018 ceph-mon[23740]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:56 smithi018 ceph-mon[28324]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:56 smithi031 ceph-mon[24497]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:09:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:58 smithi018 ceph-mon[23740]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:09:58 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:58 smithi018 ceph-mon[28324]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:09:58 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:09:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:58 smithi031 ceph-mon[24497]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:09:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:09:58 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:00 smithi018 ceph-mon[23740]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:00 smithi018 ceph-mon[23740]: overall HEALTH_OK 2024-09-21T00:10:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:00 smithi018 ceph-mon[28324]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:00 smithi018 ceph-mon[28324]: overall HEALTH_OK 2024-09-21T00:10:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:00 smithi031 ceph-mon[24497]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:00 smithi031 ceph-mon[24497]: overall HEALTH_OK 2024-09-21T00:10:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:01] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-21T00:10:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:02 smithi018 ceph-mon[23740]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:02 smithi018 ceph-mon[28324]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:02 smithi031 ceph-mon[24497]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:04 smithi018 ceph-mon[23740]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:04 smithi018 ceph-mon[28324]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:04 smithi031 ceph-mon[24497]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:06 smithi018 ceph-mon[23740]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:06 smithi018 ceph-mon[28324]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:06 smithi031 ceph-mon[24497]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:08 smithi018 ceph-mon[23740]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:08 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:08 smithi018 ceph-mon[28324]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:08 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:08 smithi031 ceph-mon[24497]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:08 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:10 smithi018 ceph-mon[23740]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:10 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:11.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:10 smithi018 ceph-mon[28324]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:11.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:10 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:10 smithi031 ceph-mon[24497]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:10 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:11] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-21T00:10:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:12 smithi018 ceph-mon[23740]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:12 smithi018 ceph-mon[28324]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:12 smithi031 ceph-mon[24497]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:14 smithi018 ceph-mon[23740]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:14 smithi018 ceph-mon[28324]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:14 smithi031 ceph-mon[24497]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:16 smithi018 ceph-mon[23740]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:16 smithi018 ceph-mon[28324]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:16 smithi031 ceph-mon[24497]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:18 smithi018 ceph-mon[23740]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:18 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:18 smithi018 ceph-mon[28324]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:18 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:18 smithi031 ceph-mon[24497]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:18 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:20 smithi018 ceph-mon[23740]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:20 smithi018 ceph-mon[28324]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:20 smithi031 ceph-mon[24497]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:21] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[28324]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[23740]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:22 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:22 smithi031 ceph-mon[24497]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:23.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:22 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:10:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:10:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:10:24.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:10:24.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:23 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:23 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:10:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:23 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:10:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:23 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:10:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:24 smithi018 ceph-mon[23740]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:24 smithi018 ceph-mon[28324]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:24 smithi031 ceph-mon[24497]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:25 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:25 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:25 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:26 smithi018 ceph-mon[23740]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:26 smithi018 ceph-mon[28324]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:26 smithi031 ceph-mon[24497]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:28 smithi018 ceph-mon[23740]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:28 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:28 smithi018 ceph-mon[28324]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:28 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:28 smithi031 ceph-mon[24497]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:28 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:30 smithi018 ceph-mon[23740]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:30 smithi018 ceph-mon[28324]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:30 smithi031 ceph-mon[24497]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:31] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-21T00:10:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:32 smithi018 ceph-mon[23740]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:32 smithi018 ceph-mon[28324]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:32 smithi031 ceph-mon[24497]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:34 smithi018 ceph-mon[23740]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:34 smithi018 ceph-mon[28324]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:34 smithi031 ceph-mon[24497]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:36 smithi018 ceph-mon[23740]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:36 smithi018 ceph-mon[28324]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:36 smithi031 ceph-mon[24497]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:38 smithi018 ceph-mon[28324]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:38 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:38 smithi018 ceph-mon[23740]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:38 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:38 smithi031 ceph-mon[24497]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:38 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:40 smithi018 ceph-mon[28324]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:40 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:40 smithi018 ceph-mon[23740]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:40 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:40 smithi031 ceph-mon[24497]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:40 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:42.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:41] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-21T00:10:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:42 smithi018 ceph-mon[23740]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:42 smithi018 ceph-mon[28324]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:42 smithi031 ceph-mon[24497]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:45.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:44 smithi018 ceph-mon[28324]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:44 smithi018 ceph-mon[23740]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:44 smithi031 ceph-mon[24497]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:46 smithi018 ceph-mon[23740]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:46 smithi018 ceph-mon[28324]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:46 smithi031 ceph-mon[24497]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:48 smithi018 ceph-mon[23740]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:48 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:48 smithi018 ceph-mon[28324]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:48 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:48 smithi031 ceph-mon[24497]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:48 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:50 smithi018 ceph-mon[23740]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:50 smithi018 ceph-mon[28324]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:50 smithi031 ceph-mon[24497]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:10:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:10:51] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-21T00:10:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:52 smithi018 ceph-mon[23740]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:52 smithi018 ceph-mon[28324]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:52 smithi031 ceph-mon[24497]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:54 smithi018 ceph-mon[28324]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:54 smithi018 ceph-mon[23740]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:54 smithi031 ceph-mon[24497]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:55 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:55 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:55 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:10:57.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:56 smithi018 ceph-mon[28324]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:56 smithi018 ceph-mon[23740]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:56 smithi031 ceph-mon[24497]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:10:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:58 smithi018 ceph-mon[23740]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:59.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:10:58 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:59.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:58 smithi018 ceph-mon[28324]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:59.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:10:58 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:10:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:58 smithi031 ceph-mon[24497]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:10:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:10:58 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:01.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:00 smithi018 ceph-mon[23740]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:00 smithi018 ceph-mon[28324]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:00 smithi031 ceph-mon[24497]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:01] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-21T00:11:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:02 smithi018 ceph-mon[23740]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:02 smithi018 ceph-mon[28324]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:03.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:02 smithi031 ceph-mon[24497]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:04 smithi018 ceph-mon[23740]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:05.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:04 smithi018 ceph-mon[28324]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:04 smithi031 ceph-mon[24497]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:06 smithi031 ceph-mon[24497]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:07.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:06 smithi018 ceph-mon[23740]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:07.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:06 smithi018 ceph-mon[28324]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:08 smithi031 ceph-mon[24497]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:08 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:08 smithi018 ceph-mon[23740]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:08 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:09.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:08 smithi018 ceph-mon[28324]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:09.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:08 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:10 smithi031 ceph-mon[24497]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:10 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:11.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:10 smithi018 ceph-mon[23740]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:11.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:10 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:11.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:10 smithi018 ceph-mon[28324]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:11.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:10 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:12.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:11] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-21T00:11:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:12 smithi031 ceph-mon[24497]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:13.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:12 smithi018 ceph-mon[28324]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:13.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:12 smithi018 ceph-mon[23740]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:15.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:14 smithi018 ceph-mon[28324]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:14 smithi018 ceph-mon[23740]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:14 smithi031 ceph-mon[24497]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:16 smithi031 ceph-mon[24497]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:17.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:16 smithi018 ceph-mon[23740]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:17.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:16 smithi018 ceph-mon[28324]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:18 smithi031 ceph-mon[24497]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:18 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:18 smithi018 ceph-mon[23740]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:18 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:18 smithi018 ceph-mon[28324]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:18 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:20 smithi031 ceph-mon[24497]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:21.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:20 smithi018 ceph-mon[23740]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:21.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:20 smithi018 ceph-mon[28324]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:21] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-21T00:11:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:22 smithi031 ceph-mon[24497]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:23.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:22 smithi018 ceph-mon[23740]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:23.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:22 smithi018 ceph-mon[28324]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:23 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:11:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:23 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:11:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:23 smithi031 ceph-mon[24497]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[23740]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:11:24.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:23 smithi018 ceph-mon[28324]: from='mgr.15186 ' entity='mgr.y' 2024-09-21T00:11:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:24 smithi031 ceph-mon[24497]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:25.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:24 smithi018 ceph-mon[23740]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:25.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:24 smithi018 ceph-mon[28324]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:25 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:25 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:25 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:26 smithi031 ceph-mon[24497]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:27.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:26 smithi018 ceph-mon[23740]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:27.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:26 smithi018 ceph-mon[28324]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:28 smithi031 ceph-mon[24497]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:28 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:29.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:28 smithi018 ceph-mon[28324]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:28 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:28 smithi018 ceph-mon[23740]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:28 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:30 smithi031 ceph-mon[24497]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:30 smithi018 ceph-mon[23740]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:30 smithi018 ceph-mon[28324]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:31] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-21T00:11:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:32 smithi031 ceph-mon[24497]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:32 smithi018 ceph-mon[28324]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:33.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:32 smithi018 ceph-mon[23740]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:34 smithi031 ceph-mon[24497]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:34 smithi018 ceph-mon[23740]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:34 smithi018 ceph-mon[28324]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:36 smithi031 ceph-mon[24497]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:36 smithi018 ceph-mon[23740]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:36 smithi018 ceph-mon[28324]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:38 smithi031 ceph-mon[24497]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:38 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:38 smithi018 ceph-mon[28324]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:38 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:38 smithi018 ceph-mon[23740]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:38 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:40 smithi031 ceph-mon[24497]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:40 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:40 smithi018 ceph-mon[23740]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:40 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:40 smithi018 ceph-mon[28324]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:40 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:41] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-21T00:11:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:42 smithi031 ceph-mon[24497]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:42 smithi018 ceph-mon[23740]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:42 smithi018 ceph-mon[28324]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:44 smithi031 ceph-mon[24497]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:44 smithi018 ceph-mon[23740]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:44 smithi018 ceph-mon[28324]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:46 smithi031 ceph-mon[24497]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:46 smithi018 ceph-mon[23740]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:46 smithi018 ceph-mon[28324]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:49 smithi018 ceph-mon[23740]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:49 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:49 smithi018 ceph-mon[28324]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:49 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:49 smithi031 ceph-mon[24497]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:49 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:51 smithi018 ceph-mon[23740]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:51 smithi018 ceph-mon[28324]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:51 smithi031 ceph-mon[24497]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:11:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:11:51] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-21T00:11:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:53 smithi018 ceph-mon[23740]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:53 smithi018 ceph-mon[28324]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:53 smithi031 ceph-mon[24497]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:55 smithi018 ceph-mon[23740]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:55 smithi018 ceph-mon[28324]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:55 smithi031 ceph-mon[24497]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:56.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:56 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:56.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:56 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:56 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:11:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:57 smithi018 ceph-mon[23740]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:57.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:57 smithi018 ceph-mon[28324]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:57 smithi031 ceph-mon[24497]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:11:58.814 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:11:59.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:59 smithi018 ceph-mon[23740]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:59.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:11:59 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:59.130 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:59 smithi018 ceph-mon[28324]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:59.130 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:11:59 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:11:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:59 smithi031 ceph-mon[24497]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:11:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:11:59 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (9m) 2m ago 17m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (8m) 2m ago 16m 51.9M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (2m) 2m ago 16m 76.4M - 3.5 e1d6a67b021e 3d9be9314aa3 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (12m) 2m ago 21m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (5m) 2m ago 23m 493M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (23m) 2m ago 23m 67.6M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:12:00.910 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (22m) 2m ago 22m 49.2M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (22m) 2m ago 22m 53.0M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (9m) 2m ago 17m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (8m) 2m ago 17m 16.7M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (21m) 2m ago 21m 54.3M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (21m) 2m ago 21m 54.0M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (20m) 2m ago 20m 52.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (20m) 2m ago 20m 52.9M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (19m) 2m ago 19m 53.9M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (19m) 2m ago 19m 51.0M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (19m) 2m ago 19m 52.7M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (18m) 2m ago 18m 54.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (2m) 2m ago 18m 43.6M - 2.43.0 a07b618ecd1d 8469e2a6a30f 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (16m) 2m ago 16m 76.4M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:12:00.911 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (16m) 2m ago 16m 77.6M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:12:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:01 smithi018 ceph-mon[23740]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:01 smithi018 ceph-mon[28324]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:01.303 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:01 smithi031 ceph-mon[24497]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:01.471 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:12:02.027 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:12:01] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-21T00:12:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:02 smithi018 ceph-mon[23740]: from='client.25177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:12:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:02 smithi018 ceph-mon[28324]: from='client.25177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:12:02.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:02 smithi031 ceph-mon[24497]: from='client.25177 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:12:03.169 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:03 smithi018 ceph-mon[23740]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:03.170 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:03 smithi018 ceph-mon[28324]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:03.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:03 smithi031 ceph-mon[24497]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:12:03.712 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:12:03.713 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:12:04.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:04 smithi018 ceph-mon[23740]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:04.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:04 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1770201984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:04.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:04 smithi018 ceph-mon[28324]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:04.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:04 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1770201984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:04.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph -s' 2024-09-21T00:12:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:04 smithi031 ceph-mon[24497]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:04 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1770201984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:06.366 INFO:teuthology.orchestra.run.smithi018.stdout: cluster: 2024-09-21T00:12:06.366 INFO:teuthology.orchestra.run.smithi018.stdout: id: c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:12:06.366 INFO:teuthology.orchestra.run.smithi018.stdout: health: HEALTH_OK 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: services: 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: mgr: y(active, since 3m), standbys: x 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: data: 2024-09-21T00:12:06.367 INFO:teuthology.orchestra.run.smithi018.stdout: pools: 6 pools, 161 pgs 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: objects: 241 objects, 585 KiB 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: usage: 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: pgs: 161 active+clean 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: io: 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-21T00:12:06.368 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T00:12:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:06 smithi018 ceph-mon[23740]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:06 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/575645843' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:12:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:06 smithi018 ceph-mon[28324]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:06 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/575645843' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:12:06.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:12:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:06 smithi031 ceph-mon[24497]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:06 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/575645843' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T00:12:09.061 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T00:12:09.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:08 smithi018 ceph-mon[23740]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:09.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:08 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:09.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:08 smithi018 ceph-mon[28324]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:09.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:08 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:08 smithi031 ceph-mon[24497]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:09.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:08 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:09.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-21T00:12:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:09 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/58447024' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:12:09.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:09 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/58447024' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:12:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:09 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/58447024' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:12:10.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:10 smithi018 ceph-mon[28324]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:10.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:10 smithi018 ceph-mon[28324]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:10.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:10 smithi018 ceph-mon[23740]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:10.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:10 smithi018 ceph-mon[23740]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:10 smithi031 ceph-mon[24497]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:10 smithi031 ceph-mon[24497]: from='mgr.15186 172.21.15.18:0/1085185772' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:11.758 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:12:12.059 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:12:11] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-21T00:12:12.060 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:11 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3757993675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:12.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:11 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3757993675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:11 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3757993675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:12:12.359 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph mgr fail' 2024-09-21T00:12:12.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:12 smithi018 ceph-mon[23740]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:12.918 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:12 smithi018 ceph-mon[28324]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:12 smithi031 ceph-mon[24497]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:12:15.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[23740]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:15.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:12:15.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[23740]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[28324]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mon[28324]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ignoring --setuser ceph since I am not root 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ignoring --setgroup ceph since I am not root 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-mgr[61431]: -- 172.21.15.18:0/1763437308 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cf87e230e0 con 0x55cf87e01000 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:14.980+0000 7fe78b989040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:12:15.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.039+0000 7fe78b989040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:12:15.240 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:14] ENGINE Bus STOPPING 2024-09-21T00:12:15.240 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:14 smithi031 ceph-mon[24497]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:15.240 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:14 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-21T00:12:15.240 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:14 smithi031 ceph-mon[24497]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T00:12:15.424 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.199+0000 7fe78b989040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:12:15.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.269+0000 7fe78b989040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:12:15.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 180' 2024-09-21T00:12:15.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:12:15.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:15] ENGINE Bus STOPPED 2024-09-21T00:12:15.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:15] ENGINE Bus STARTING 2024-09-21T00:12:15.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:15] ENGINE Serving on http://:::9283 2024-09-21T00:12:15.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:12:15] ENGINE Bus STARTED 2024-09-21T00:12:15.678 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.425+0000 7fe78b989040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:12:15.678 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.578+0000 7fe78b989040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:12:15.933 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: mgrmap e32: x(active, starting, since 0.35896s) 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:12:15.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:12:15.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: Manager daemon x is now available 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:15.936 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:15.937 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:15.937 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:15.937 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:12:15.937 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: mgrmap e32: x(active, starting, since 0.35896s) 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:12:15.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: Manager daemon x is now available 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:15.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:15.940 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:16.202 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:15.931+0000 7fe78b989040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:12:16.202 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:16.002+0000 7fe78b989040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:12:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/840167050' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-21T00:12:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: mgrmap e32: x(active, starting, since 0.35896s) 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:12:16.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: Manager daemon x is now available 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:16.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:15 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T00:12:16.595 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:16.208+0000 7fe78b989040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:12:16.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-mon[23740]: mgrmap e33: x(active, since 1.36367s) 2024-09-21T00:12:16.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-mon[23740]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:16.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-mon[28324]: mgrmap e33: x(active, since 1.36367s) 2024-09-21T00:12:16.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:16 smithi018 ceph-mon[28324]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:17.176 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:16 smithi031 ceph-mon[24497]: mgrmap e33: x(active, since 1.36367s) 2024-09-21T00:12:17.176 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:16 smithi031 ceph-mon[24497]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:17.403 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:17.206+0000 7fe78b989040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:12:17.404 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:17.278+0000 7fe78b989040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:12:18.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-mon[24497]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:18.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-mon[24497]: mgrmap e34: x(active, since 2s) 2024-09-21T00:12:18.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:12:17.735+0000 7f8e69298640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:00:12:17] ENGINE Error in HTTPServer.serve 2024-09-21T00:12:18.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._connections.run(self.expiration_interval) 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._run(expiration_interval) 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: s = self.context.wrap_socket( 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.sslsocket_class._create( 2024-09-21T00:12:18.002 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:12:18.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.do_handshake() 2024-09-21T00:12:18.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:12:18.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self._sslobj.do_handshake() 2024-09-21T00:12:18.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:12:18.003 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:12:18.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-mon[28324]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:18.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-mon[28324]: mgrmap e34: x(active, since 2s) 2024-09-21T00:12:18.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-mon[23740]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:18.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:17 smithi018 ceph-mon[23740]: mgrmap e34: x(active, since 2s) 2024-09-21T00:12:18.365 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.076+0000 7fe78b989040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:12:18.366 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.214+0000 7fe78b989040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:12:18.366 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.364+0000 7fe78b989040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:12:18.664 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.430+0000 7fe78b989040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:12:18.664 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.590+0000 7fe78b989040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:12:18.665 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.662+0000 7fe78b989040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.726+0000 7fe78b989040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.906+0000 7fe78b989040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: [21/Sep/2024:00:12:17] ENGINE Bus STARTING 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: [21/Sep/2024:00:12:17] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: [21/Sep/2024:00:12:17] ENGINE Error in HTTPServer.serve 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:12:18.925 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.do_handshake() 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:12:18.926 INFO:journalctl@ceph.mon.a.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: [21/Sep/2024:00:12:17] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[23740]: [21/Sep/2024:00:12:17] ENGINE Bus STARTED 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: [21/Sep/2024:00:12:17] ENGINE Bus STARTING 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: [21/Sep/2024:00:12:17] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: [21/Sep/2024:00:12:17] ENGINE Error in HTTPServer.serve 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:12:18.927 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.do_handshake() 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: [21/Sep/2024:00:12:17] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:12:18.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-mon[28324]: [21/Sep/2024:00:12:17] ENGINE Bus STARTED 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: [21/Sep/2024:00:12:17] ENGINE Bus STARTING 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: [21/Sep/2024:00:12:17] ENGINE Serving on https://172.21.15.31:7150 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: [21/Sep/2024:00:12:17] ENGINE Error in HTTPServer.serve 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._run(expiration_interval) 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:12:19.106 INFO:journalctl@ceph.mon.b.smithi031.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: s = self.context.wrap_socket( 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.sslsocket_class._create( 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.do_handshake() 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._sslobj.do_handshake() 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: from='client.24995 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:19.107 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: [21/Sep/2024:00:12:17] ENGINE Serving on http://172.21.15.31:8765 2024-09-21T00:12:19.108 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:18 smithi031 ceph-mon[24497]: [21/Sep/2024:00:12:17] ENGINE Bus STARTED 2024-09-21T00:12:19.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:18 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:18.984+0000 7fe78b989040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:12:19.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:19.073+0000 7fe78b989040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:12:19.251 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:19.187+0000 7fe78b989040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:12:19.572 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:19.249+0000 7fe78b989040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:12:19.573 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:19.311+0000 7fe78b989040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:12:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:19 smithi031 ceph-mon[24497]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:19 smithi031 ceph-mon[24497]: mgrmap e35: x(active, since 4s) 2024-09-21T00:12:20.172 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-mon[23740]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:20.172 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-mon[23740]: mgrmap e35: x(active, since 4s) 2024-09-21T00:12:20.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-mon[28324]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:20.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:19 smithi018 ceph-mon[28324]: mgrmap e35: x(active, since 4s) 2024-09-21T00:12:20.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:20.170+0000 7fe78b989040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:12:20.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:12:20] ENGINE Bus STARTING 2024-09-21T00:12:20.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: CherryPy Checker: 2024-09-21T00:12:20.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: The Application mounted at '' has an empty config. 2024-09-21T00:12:20.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:12:20.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:12:20] ENGINE Serving on http://:::9283 2024-09-21T00:12:20.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:12:20] ENGINE Bus STARTED 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: Standby manager daemon y started 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:12:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: Standby manager daemon y started 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:12:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:12:21.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:20 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: Standby manager daemon y started 2024-09-21T00:12:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T00:12:21.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:12:21.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T00:12:21.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:12:21.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:21.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:20 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:12:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[23740]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[23740]: mgrmap e36: x(active, since 6s), standbys: y 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[28324]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-09-21T00:12:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[28324]: mgrmap e36: x(active, since 6s), standbys: y 2024-09-21T00:12:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:12:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:21 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:21 smithi031 ceph-mon[24497]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-09-21T00:12:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:21 smithi031 ceph-mon[24497]: mgrmap e36: x(active, since 6s), standbys: y 2024-09-21T00:12:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:21 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:12:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:21 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:21 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:12:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:23.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:12:23.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:23.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:23 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:24.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:24.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:24.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:23 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:24.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:24.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:24.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:24.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:24.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:24.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:24.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:24.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:24 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:24 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:24 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:12:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:24 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:25.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:24 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:25.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:25 smithi018 ceph-mon[23740]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:12:25.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:25 smithi018 ceph-mon[28324]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:12:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:25 smithi031 ceph-mon[24497]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-21T00:12:26.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:12:26.361+0000 7f8ea0d84640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:26.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:12:26.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:26.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.wait_async( 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return future.result(timeout) 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:26.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.__get_result() 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise self._exception 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: out, err, code = await self._run_cephadm( 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise OrchestratorError( 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:26.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:26.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:12:28.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[23740]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[23740]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:12:28.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[23740]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:28.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[28324]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[28324]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:27 smithi018 ceph-mon[28324]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:27 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.185 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.186 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:28.187 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:28.188 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:28.189 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:28.190 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:28.190 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:28.190 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:27 smithi031 ceph-mon[24497]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:12:28.190 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:27 smithi031 ceph-mon[24497]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:12:28.190 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:27 smithi031 ceph-mon[24497]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:12:29.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 systemd[1]: Stopping Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:12:30.112 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-mon[24497]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T00:12:30.113 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.845Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.846Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.846Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.846Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[53649]: ts=2024-09-21T00:12:29.846Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T00:12:30.114 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:29 smithi031 podman[56572]: 2024-09-21 00:12:29.855844947 +0000 UTC m=+0.150231412 container died 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:30.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:29 smithi018 ceph-mon[23740]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:30.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:29 smithi018 ceph-mon[28324]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:30.497 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 podman[56572]: 2024-09-21 00:12:30.211964238 +0000 UTC m=+0.506350700 container cleanup 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:30.497 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 bash[56572]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-21T00:12:30.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 podman[56586]: 2024-09-21 00:12:30.496507018 +0000 UTC m=+0.639656076 container remove 8469e2a6a30fbdd19379ff65f8dffc18c310dc0ba5c61e4b94fee3535a3cd77d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:30.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T00:12:30.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 systemd[1]: Stopped Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:12:30.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Consumed 1.758s CPU time. 2024-09-21T00:12:30.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 systemd[1]: Starting Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:12:31.108 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:30 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:31.109 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 podman[56676]: 2024-09-21 00:12:30.750668342 +0000 UTC m=+0.017534511 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T00:12:31.109 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:30 smithi031 podman[56676]: 2024-09-21 00:12:30.881363066 +0000 UTC m=+0.148229226 container create a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:31.109 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 podman[56676]: 2024-09-21 00:12:31.108547295 +0000 UTC m=+0.375413469 container init a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:30 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:30 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:31.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 podman[56676]: 2024-09-21 00:12:31.112449041 +0000 UTC m=+0.379315205 container start a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.136Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.136Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.136Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi031 (none))" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.136Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.136Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.138Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.138Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.140Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.141Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.142Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.143Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.304592ms 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.143Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T00:12:31.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.151Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.166Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.174Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.176Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.177Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.177Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.508µs wal_replay_duration=33.429317ms wbl_replay_duration=120ns total_replay_duration=34.784412ms 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.180Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.180Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.180Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.202Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.633951ms db_storage=817ns remote_storage=1.122µs web_handler=361ns query_engine=670ns scrape=274.995µs scrape_sd=115.173µs notify=22.97µs notify_sd=19.076µs rules=20.717288ms tracing=6.481µs 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.202Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:12:31.202Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 bash[56676]: a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f 2024-09-21T00:12:31.503 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:12:31 smithi031 systemd[1]: Started Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:12:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[23740]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:32.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[28324]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:32.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:31 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-mon[24497]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:31 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:33.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:32 smithi031 ceph-mon[24497]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:32 smithi018 ceph-mon[23740]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:32 smithi018 ceph-mon[28324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:12:34.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:33 smithi018 ceph-mon[23740]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:34.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:33 smithi018 ceph-mon[28324]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:34.203 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:33 smithi031 ceph-mon[24497]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:36 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:36.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:12:36.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:36 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:12:37.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:37 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:37.382 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:37 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:37.382 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:37 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:37.382 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:37 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:37 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:12:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:37 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:12:38.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:38 smithi018 ceph-mon[23740]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:38.304 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:38 smithi018 ceph-mon[28324]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:38.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:38 smithi031 ceph-mon[24497]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:12:39.166+0000 7f8ea0d84640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:39.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:39.503 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:39.504 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.wait_async( 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return future.result(timeout) 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.__get_result() 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise self._exception 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: out, err, code = await self._run_cephadm( 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise OrchestratorError( 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:39.505 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:39.506 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:39.507 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:40.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[28324]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/284663969' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:12:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]: dispatch 2024-09-21T00:12:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[23740]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/284663969' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:12:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:40 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]: dispatch 2024-09-21T00:12:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:40 smithi031 ceph-mon[24497]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:40 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/284663969' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:12:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:40 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]: dispatch 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.426 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:41.427 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.428 INFO:journalctl@ceph.mon.a.smithi018.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.429 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.430 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.431 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:12:41.432 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.433 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]': finished 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[23740]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]: dispatch 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.435 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]': finished 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[28324]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T00:12:41.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:41 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]: dispatch 2024-09-21T00:12:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:41 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.503 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:12:41.504 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:12:41.505 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi-foo-smithi018-wbwhhs 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: /usr/bin/podman: stderr Error: no such container ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.506 INFO:journalctl@ceph.mon.b.smithi031.stdout: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:12:41.507 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:12:41.508 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:41 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/4066741734' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1287191520"}]': finished 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:41 smithi031 ceph-mon[24497]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T00:12:41.509 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:41 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]: dispatch 2024-09-21T00:12:42.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[23740]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]': finished 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[23740]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3158124834' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[28324]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]': finished 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[28324]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3158124834' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:42 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:42 smithi031 ceph-mon[24497]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-09-21T00:12:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1750688457' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1287191520"}]': finished 2024-09-21T00:12:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:42 smithi031 ceph-mon[24497]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T00:12:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3158124834' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:42 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]: dispatch 2024-09-21T00:12:43.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:43 smithi031 ceph-mon[24497]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]': finished 2024-09-21T00:12:43.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:43 smithi031 ceph-mon[24497]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T00:12:43.350 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:43 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]: dispatch 2024-09-21T00:12:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[28324]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]': finished 2024-09-21T00:12:43.531 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[28324]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T00:12:43.531 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]: dispatch 2024-09-21T00:12:43.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[23740]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/45415870"}]': finished 2024-09-21T00:12:43.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[23740]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T00:12:43.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:43 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]: dispatch 2024-09-21T00:12:43.674 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:12:43] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[23740]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]': finished 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[23740]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]: dispatch 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[28324]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]': finished 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[28324]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T00:12:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:44 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]: dispatch 2024-09-21T00:12:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:44 smithi031 ceph-mon[24497]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-21T00:12:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:44 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2358329943' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3414036052"}]': finished 2024-09-21T00:12:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:44 smithi031 ceph-mon[24497]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T00:12:44.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:44 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]: dispatch 2024-09-21T00:12:45.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]': finished 2024-09-21T00:12:45.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[28324]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T00:12:45.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]: dispatch 2024-09-21T00:12:45.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]': finished 2024-09-21T00:12:45.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[23740]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T00:12:45.327 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]: dispatch 2024-09-21T00:12:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2552111713' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3699305031"}]': finished 2024-09-21T00:12:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:45 smithi031 ceph-mon[24497]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T00:12:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]: dispatch 2024-09-21T00:12:46.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[23740]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:46.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]': finished 2024-09-21T00:12:46.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[23740]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T00:12:46.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:46.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[28324]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:46.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]': finished 2024-09-21T00:12:46.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[28324]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T00:12:46.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:46 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:46 smithi031 ceph-mon[24497]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:12:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:46 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1062696217' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1249860975"}]': finished 2024-09-21T00:12:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:46 smithi031 ceph-mon[24497]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T00:12:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:46 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:12:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:48 smithi018 ceph-mon[23740]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:12:48.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:48 smithi018 ceph-mon[28324]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:12:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:48 smithi031 ceph-mon[24497]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:12:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:50 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:50 smithi018 ceph-mon[23740]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:50 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:50.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:50 smithi018 ceph-mon[28324]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:50 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:12:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:50 smithi031 ceph-mon[24497]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:12:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:51 smithi018 ceph-mon[23740]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:12:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:51 smithi018 ceph-mon[28324]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:12:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:51 smithi031 ceph-mon[24497]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:12:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:12:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:12:53] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-21T00:12:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:53 smithi018 ceph-mon[23740]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:12:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:53 smithi018 ceph-mon[28324]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:12:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:53 smithi031 ceph-mon[24497]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:12:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:55 smithi018 ceph-mon[23740]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:12:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:55 smithi018 ceph-mon[28324]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:12:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:55 smithi031 ceph-mon[24497]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:12:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:57 smithi018 ceph-mon[23740]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:12:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:57 smithi018 ceph-mon[28324]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:12:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:57 smithi031 ceph-mon[24497]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:13:00.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:59 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:00.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:12:59 smithi018 ceph-mon[23740]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:00.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:59 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:00.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:12:59 smithi018 ceph-mon[28324]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:59 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:12:59 smithi031 ceph-mon[24497]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:01.166 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:00 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:00 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:00 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:01 smithi018 ceph-mon[23740]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:01 smithi018 ceph-mon[28324]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:01 smithi031 ceph-mon[24497]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:03] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-21T00:13:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:03 smithi018 ceph-mon[23740]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:04.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:03 smithi018 ceph-mon[28324]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:03 smithi031 ceph-mon[24497]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:05 smithi018 ceph-mon[23740]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:05 smithi018 ceph-mon[28324]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:05 smithi031 ceph-mon[24497]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:07 smithi018 ceph-mon[23740]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:07 smithi018 ceph-mon[28324]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:07 smithi031 ceph-mon[24497]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:10.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:09 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:10.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:09 smithi018 ceph-mon[23740]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:09 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:09 smithi018 ceph-mon[28324]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:09 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:09 smithi031 ceph-mon[24497]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:11 smithi018 ceph-mon[23740]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:11 smithi018 ceph-mon[28324]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:11 smithi031 ceph-mon[24497]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:13] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-21T00:13:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:13 smithi018 ceph-mon[23740]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:13 smithi018 ceph-mon[28324]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:13 smithi031 ceph-mon[24497]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:15 smithi018 ceph-mon[23740]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:15 smithi018 ceph-mon[28324]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:15 smithi031 ceph-mon[24497]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-21T00:13:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:17 smithi018 ceph-mon[23740]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:17 smithi018 ceph-mon[28324]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:17 smithi031 ceph-mon[24497]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:19 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:19 smithi018 ceph-mon[23740]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:19 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:19 smithi018 ceph-mon[28324]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:19 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:19 smithi031 ceph-mon[24497]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:21 smithi018 ceph-mon[23740]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:21 smithi018 ceph-mon[28324]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:21 smithi031 ceph-mon[24497]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:23] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-21T00:13:24.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:23 smithi018 ceph-mon[23740]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:23 smithi018 ceph-mon[28324]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:23 smithi031 ceph-mon[24497]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:25 smithi018 ceph-mon[23740]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:25 smithi018 ceph-mon[28324]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:25 smithi031 ceph-mon[24497]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:27 smithi018 ceph-mon[23740]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:28.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:27 smithi018 ceph-mon[28324]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:27 smithi031 ceph-mon[24497]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:30.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:29 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:30.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:29 smithi018 ceph-mon[23740]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:30.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:29 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:30.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:29 smithi018 ceph-mon[28324]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:29 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:29 smithi031 ceph-mon[24497]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:30 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:30 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:30 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:31 smithi018 ceph-mon[23740]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:32.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:31 smithi018 ceph-mon[28324]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:31 smithi031 ceph-mon[24497]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:33] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-21T00:13:34.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:33 smithi018 ceph-mon[23740]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:34.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:33 smithi018 ceph-mon[28324]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:33 smithi031 ceph-mon[24497]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:35 smithi018 ceph-mon[23740]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:35 smithi018 ceph-mon[28324]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:35 smithi031 ceph-mon[24497]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:38.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:37 smithi031 ceph-mon[24497]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:37 smithi018 ceph-mon[23740]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:38.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:37 smithi018 ceph-mon[28324]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:40.213 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:40.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:39 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:13:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:39 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:13:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:40 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:40 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:13:41.161+0000 7f8ea0d84640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:41.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:41.253 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.wait_async( 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return future.result(timeout) 2024-09-21T00:13:41.254 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.__get_result() 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise self._exception 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: out, err, code = await self._run_cephadm( 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise OrchestratorError( 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:41.255 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:41.256 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:40 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:13:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:40 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:13:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:40 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:13:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:40 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:13:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:41 smithi031 ceph-mon[24497]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:42.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:41 smithi018 ceph-mon[23740]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:41 smithi018 ceph-mon[28324]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:42 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:13:43.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.256 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:42 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:13:43.427 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.428 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.429 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:42 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.430 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:13:43.431 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:13:43.432 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:13:43.433 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:13:43.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:13:43.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:13:43.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:13:43.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:13:43.434 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:13:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:43] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-21T00:13:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:43 smithi031 ceph-mon[24497]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:43 smithi018 ceph-mon[28324]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:43 smithi018 ceph-mon[23740]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:46.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:45 smithi018 ceph-mon[23740]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:46.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:45 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:46.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:45 smithi018 ceph-mon[28324]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:46.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:45 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:45 smithi031 ceph-mon[24497]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:46.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:45 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:13:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:47 smithi031 ceph-mon[24497]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:47 smithi018 ceph-mon[23740]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:48.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:47 smithi018 ceph-mon[28324]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:49 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:49 smithi031 ceph-mon[24497]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:50.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:49 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:49 smithi018 ceph-mon[28324]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:49 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:13:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:49 smithi018 ceph-mon[23740]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:51 smithi018 ceph-mon[23740]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:51 smithi018 ceph-mon[28324]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:51 smithi031 ceph-mon[24497]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:13:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:13:53] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-21T00:13:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:53 smithi018 ceph-mon[23740]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:53 smithi018 ceph-mon[28324]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:53 smithi031 ceph-mon[24497]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:55 smithi018 ceph-mon[23740]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:55 smithi018 ceph-mon[28324]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:55 smithi031 ceph-mon[24497]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:13:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:57 smithi018 ceph-mon[23740]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:57 smithi018 ceph-mon[28324]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:13:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:57 smithi031 ceph-mon[24497]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:00.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:59 smithi018 ceph-mon[23740]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:00.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:13:59 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:59 smithi018 ceph-mon[28324]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:00.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:13:59 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:59 smithi031 ceph-mon[24497]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:13:59 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:00 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:00 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:00 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:01 smithi031 ceph-mon[24497]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:01 smithi018 ceph-mon[23740]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:01 smithi018 ceph-mon[28324]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:03] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-21T00:14:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:03 smithi031 ceph-mon[24497]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:03 smithi018 ceph-mon[23740]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:04.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:03 smithi018 ceph-mon[28324]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:05 smithi031 ceph-mon[24497]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:06.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:05 smithi018 ceph-mon[23740]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:05 smithi018 ceph-mon[28324]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:07 smithi031 ceph-mon[24497]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:08.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:07 smithi018 ceph-mon[23740]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:08.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:07 smithi018 ceph-mon[28324]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:09 smithi031 ceph-mon[24497]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:09 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:10.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:09 smithi018 ceph-mon[23740]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:10.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:09 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:10.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:09 smithi018 ceph-mon[28324]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:10.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:09 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:11 smithi018 ceph-mon[23740]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:11 smithi018 ceph-mon[28324]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:11 smithi031 ceph-mon[24497]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:13] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.43.0" 2024-09-21T00:14:14.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:13 smithi018 ceph-mon[23740]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:13 smithi018 ceph-mon[28324]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:13 smithi031 ceph-mon[24497]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:16.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:15 smithi018 ceph-mon[23740]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:15 smithi018 ceph-mon[28324]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:15 smithi031 ceph-mon[24497]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:17 smithi031 ceph-mon[24497]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:17 smithi018 ceph-mon[23740]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:18.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:17 smithi018 ceph-mon[28324]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:19 smithi031 ceph-mon[24497]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:19 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:19 smithi018 ceph-mon[23740]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:19 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:19 smithi018 ceph-mon[28324]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:19 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:21 smithi018 ceph-mon[23740]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:21 smithi018 ceph-mon[28324]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:21 smithi031 ceph-mon[24497]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:23] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.43.0" 2024-09-21T00:14:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:23 smithi018 ceph-mon[23740]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:23 smithi018 ceph-mon[28324]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:23 smithi031 ceph-mon[24497]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:25 smithi031 ceph-mon[24497]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:25 smithi018 ceph-mon[23740]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:25 smithi018 ceph-mon[28324]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:27 smithi031 ceph-mon[24497]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:28.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:27 smithi018 ceph-mon[23740]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:27 smithi018 ceph-mon[28324]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:29 smithi031 ceph-mon[24497]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:29 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:30.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:29 smithi018 ceph-mon[28324]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:30.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:29 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:29 smithi018 ceph-mon[23740]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:29 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:30 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:30 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:30 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:31 smithi031 ceph-mon[24497]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:32.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:31 smithi018 ceph-mon[28324]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:32.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:31 smithi018 ceph-mon[23740]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:33] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-21T00:14:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:33 smithi031 ceph-mon[24497]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:34.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:33 smithi018 ceph-mon[28324]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:33 smithi018 ceph-mon[23740]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:35 smithi031 ceph-mon[24497]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:35 smithi018 ceph-mon[23740]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:35 smithi018 ceph-mon[28324]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:38.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:37 smithi031 ceph-mon[24497]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:37 smithi018 ceph-mon[23740]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:38.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:37 smithi018 ceph-mon[28324]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:39 smithi031 ceph-mon[24497]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:39 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:39 smithi018 ceph-mon[23740]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:39 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:39 smithi018 ceph-mon[28324]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:39 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:41 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:14:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:14:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:14:42.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:41 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:14:43.106 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:42 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:14:43.106 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:42 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:14:43.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:42 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:14:43.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:42 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:14:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:42 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:14:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:42 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:43] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:14:43.352+0000 7f8ea0d84640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:43.752 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:43.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.wait_async( 2024-09-21T00:14:43.754 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return future.result(timeout) 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.__get_result() 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise self._exception 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: out, err, code = await self._run_cephadm( 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise OrchestratorError( 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:43.755 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:43.756 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:43.757 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:43.757 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:43.757 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:43.757 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:43.757 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:44.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:43 smithi018 ceph-mon[23740]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:44.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:43 smithi018 ceph-mon[28324]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:43 smithi031 ceph-mon[24497]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:44 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:14:45.177 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:44 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:14:45.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:14:45.183 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:14:45.184 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.185 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:44 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:14:45.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:14:45.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:14:45.256 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:14:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:45 smithi018 ceph-mon[23740]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:45 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:45 smithi018 ceph-mon[28324]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:45 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:45 smithi031 ceph-mon[24497]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:45 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:14:48.235 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:47 smithi018 ceph-mon[23740]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:48.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:47 smithi018 ceph-mon[28324]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:47 smithi031 ceph-mon[24497]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:49 smithi031 ceph-mon[24497]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:49 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:49 smithi018 ceph-mon[23740]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:49 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:49 smithi018 ceph-mon[28324]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:49 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:14:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:51 smithi018 ceph-mon[23740]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:51 smithi018 ceph-mon[28324]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:51 smithi031 ceph-mon[24497]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:53.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:14:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:14:53] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-21T00:14:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:53 smithi031 ceph-mon[24497]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:54.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:53 smithi018 ceph-mon[23740]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:54.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:53 smithi018 ceph-mon[28324]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:55 smithi031 ceph-mon[24497]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:55 smithi018 ceph-mon[23740]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:56.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:55 smithi018 ceph-mon[28324]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:14:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:57 smithi031 ceph-mon[24497]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:57 smithi018 ceph-mon[23740]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:14:58.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:57 smithi018 ceph-mon[28324]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:59 smithi031 ceph-mon[24497]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:14:59 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:59 smithi018 ceph-mon[23740]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:14:59 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:00.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:59 smithi018 ceph-mon[28324]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:00.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:14:59 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:00 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:01.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:00 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:01.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:00 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:01 smithi031 ceph-mon[24497]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:01 smithi018 ceph-mon[23740]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:01 smithi018 ceph-mon[28324]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:03.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:15:03] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-21T00:15:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:03 smithi031 ceph-mon[24497]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:03 smithi018 ceph-mon[23740]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:04.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:03 smithi018 ceph-mon[28324]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:05 smithi031 ceph-mon[24497]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:05 smithi018 ceph-mon[28324]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:05 smithi018 ceph-mon[23740]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:07 smithi031 ceph-mon[24497]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:08.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:07 smithi018 ceph-mon[28324]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:08.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:07 smithi018 ceph-mon[23740]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:09 smithi031 ceph-mon[24497]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:09 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:10.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:09 smithi018 ceph-mon[23740]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:10.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:09 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:10.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:09 smithi018 ceph-mon[28324]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:10.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:09 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:11 smithi018 ceph-mon[23740]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:11 smithi018 ceph-mon[28324]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:11 smithi031 ceph-mon[24497]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:13.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:15:13] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-21T00:15:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:13 smithi031 ceph-mon[24497]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:14.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:13 smithi018 ceph-mon[23740]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:13 smithi018 ceph-mon[28324]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:15 smithi031 ceph-mon[24497]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:15 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:15 smithi018 ceph-mon[23740]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:15 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:15 smithi018 ceph-mon[28324]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:15 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:18.030 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:15:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:17 smithi031 ceph-mon[24497]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:18.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:17 smithi018 ceph-mon[23740]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:18.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:17 smithi018 ceph-mon[28324]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (12m) 2m ago 20m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (12m) 2m ago 20m 52.0M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (6m) 2m ago 19m 51.3M - 3.5 e1d6a67b021e 3d9be9314aa3 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (15m) 2m ago 25m 491M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (9m) 2m ago 26m 145M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:15:20.090 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (26m) 2m ago 26m 71.0M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (25m) 2m ago 25m 52.4M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (25m) 2m ago 25m 51.0M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (12m) 2m ago 21m 16.3M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (12m) 2m ago 21m 16.5M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (24m) 2m ago 24m 54.3M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (24m) 2m ago 24m 54.0M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (24m) 2m ago 24m 52.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (23m) 2m ago 23m 53.0M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (23m) 2m ago 23m 54.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (22m) 2m ago 22m 51.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (22m) 2m ago 22m 52.8M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (22m) 2m ago 22m 54.6M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (2m) 2m ago 21m 42.4M - 2.43.0 a07b618ecd1d a129a7d438ea 2024-09-21T00:15:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (19m) 2m ago 19m 77.1M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:15:20.092 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (19m) 2m ago 19m 78.6M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:15:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:19 smithi031 ceph-mon[24497]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:19 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:19 smithi018 ceph-mon[23740]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:19 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:19 smithi018 ceph-mon[28324]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:19 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:20.823 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ls' 2024-09-21T00:15:21.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:20 smithi018 ceph-mon[23740]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:21.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:20 smithi018 ceph-mon[28324]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:20 smithi031 ceph-mon[24497]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:21 smithi031 ceph-mon[24497]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:22.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:21 smithi018 ceph-mon[23740]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:22.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:21 smithi018 ceph-mon[28324]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:22.882 INFO:teuthology.orchestra.run.smithi018.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-21T00:15:22.882 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi018=a;count:1 2024-09-21T00:15:22.882 INFO:teuthology.orchestra.run.smithi018.stdout:grafana ?:3000 1/1 2m ago 21m smithi031=a;count:1 2024-09-21T00:15:22.882 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo ?:5000 1/1 3m ago 19m count:1 2024-09-21T00:15:22.882 INFO:teuthology.orchestra.run.smithi018.stdout:mgr 2/2 3m ago 25m smithi018=y;smithi031=x;count:2 2024-09-21T00:15:22.883 INFO:teuthology.orchestra.run.smithi018.stdout:mon 3/3 3m ago 25m smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi031:172.21.15.31=b;count:3 2024-09-21T00:15:22.883 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi018=a;smithi031=b;count:2 2024-09-21T00:15:22.883 INFO:teuthology.orchestra.run.smithi018.stdout:osd 8 3m ago - 2024-09-21T00:15:22.883 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus ?:9095 1/1 2m ago 21m smithi031=a;count:1 2024-09-21T00:15:22.883 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-09-21T00:15:23.489 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:15:23.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:15:23] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-21T00:15:24.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:23 smithi018 ceph-mon[23740]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:24.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:23 smithi018 ceph-mon[23740]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:24.053 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:23 smithi018 ceph-mon[28324]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:24.053 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:23 smithi018 ceph-mon[28324]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:23 smithi031 ceph-mon[24497]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:23 smithi031 ceph-mon[24497]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:25.645 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:15:25.645 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:15:25.645 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:15:25.646 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:15:25.647 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:15:25.647 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:15:25.647 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:15:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:25 smithi018 ceph-mon[28324]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:25 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/724117859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:26.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:25 smithi018 ceph-mon[23740]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:26.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:25 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/724117859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:25 smithi031 ceph-mon[24497]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:25 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/724117859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:26.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-21T00:15:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:27 smithi031 ceph-mon[24497]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:28.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:27 smithi018 ceph-mon[23740]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:27 smithi018 ceph-mon[28324]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:29.422 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.040 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.041 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:29 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:30.063 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:15:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:15:30.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:29 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[23740]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[23740]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:15:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[23740]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:15:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:31.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[28324]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:31.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[28324]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:15:31.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[28324]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:15:31.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:30 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:30 smithi031 ceph-mon[24497]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:30 smithi031 ceph-mon[24497]: Reconfiguring iscsi.foo.smithi018.wbwhhs (dependencies changed)... 2024-09-21T00:15:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:30 smithi031 ceph-mon[24497]: Reconfiguring daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:15:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:30 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:31.564+0000 7f8ea0d84640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:31.967 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:31.968 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.wait_async( 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:15:31.969 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return future.result(timeout) 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return self.__get_result() 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise self._exception 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: out, err, code = await self._run_cephadm( 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: raise OrchestratorError( 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Creating ceph-iscsi config... 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:31.970 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: Traceback (most recent call last): 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: return _run_code(code, main_globals, None, 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: exec(code, run_globals) 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:31.971 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:31.972 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:31.972 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:31.972 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:31.972 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:32.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:31 smithi018 ceph-mon[23740]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:32.096 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:31 smithi018 ceph-mon[28324]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:31 smithi031 ceph-mon[24497]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:32 smithi018 ceph-mon[28324]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.176 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.wait_async( 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: return future.result(timeout) 2024-09-21T00:15:33.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:32 smithi018 ceph-mon[23740]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:15:33.179 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.wait_async( 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: return future.result(timeout) 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.__get_result() 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise self._exception 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:15:33.180 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.181 INFO:journalctl@ceph.mon.c.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:32 smithi018 ceph-mon[28324]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:33.182 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.__get_result() 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise self._exception 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: raise OrchestratorError( 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.183 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.184 INFO:journalctl@ceph.mon.a.smithi018.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:32 smithi018 ceph-mon[23740]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:32 smithi031 ceph-mon[24497]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.252 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.wait_async( 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return future.result(timeout) 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.__get_result() 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise self._exception 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-21T00:15:33.253 INFO:journalctl@ceph.mon.b.smithi031.stdout: out, err, code = await self._run_cephadm( 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: raise OrchestratorError( 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Creating ceph-iscsi config... 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Non-zero exit code 1 from systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:15:33.254 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: exec(code, run_globals) 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: RuntimeError: Failed command: systemctl restart ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout: See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:15:33.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:32 smithi031 ceph-mon[24497]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:33.441 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:15:33.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:15:33] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.43.0" 2024-09-21T00:15:34.143 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (12m) 3m ago 20m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (12m) 2m ago 20m 52.0M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 running (6m) 3m ago 19m 51.3M - 3.5 e1d6a67b021e 3d9be9314aa3 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283 running (15m) 2m ago 25m 491M - 19.1.1-367-g9e2ac70b cc9251b744a3 1c08857ae5b7 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (9m) 3m ago 26m 145M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (26m) 3m ago 27m 71.0M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (25m) 2m ago 25m 52.4M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (25m) 3m ago 25m 51.0M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (12m) 3m ago 21m 16.3M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (12m) 2m ago 21m 16.5M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (25m) 3m ago 25m 54.3M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (24m) 3m ago 24m 54.0M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (24m) 3m ago 24m 52.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:15:34.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (23m) 3m ago 23m 53.0M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (23m) 2m ago 23m 54.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (23m) 2m ago 23m 51.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (22m) 2m ago 22m 52.8M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (22m) 2m ago 22m 54.6M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (3m) 2m ago 21m 42.4M - 2.43.0 a07b618ecd1d a129a7d438ea 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (20m) 3m ago 20m 77.1M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:15:34.145 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (19m) 2m ago 19m 78.6M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:15:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:33 smithi031 ceph-mon[24497]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:34.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:33 smithi018 ceph-mon[23740]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:33 smithi018 ceph-mon[28324]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:34.557 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:15:34.558 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:15:34.559 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:15:34.914 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:15:34.914 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr" 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 image", 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:15:34.915 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:15:35.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE Bus STOPPING 2024-09-21T00:15:35.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:15:35.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE Bus STOPPED 2024-09-21T00:15:35.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE Bus STARTING 2024-09-21T00:15:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:35.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Failing over to other MGR 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:15:35.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3189674852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Failing over to other MGR 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:15:35.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-21T00:15:35.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3189674852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:35.401 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.019+0000 7f8f0bab7640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T00:15:35.401 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ignoring --setuser ceph since I am not root 2024-09-21T00:15:35.401 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ignoring --setgroup ceph since I am not root 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mgr[44199]: -- 172.21.15.31:0/1905924048 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558ea04ad0e0 con 0x558ea048b000 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.177+0000 7fd98d66e040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.236+0000 7fd98d66e040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.399+0000 7fd98d66e040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:15:35.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Failing over to other MGR 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='mgr.24956 172.21.15.31:0/143272040' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T00:15:35.404 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:15:35.404 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-21T00:15:35.404 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3189674852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:15:35.562 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE Serving on http://:::9283 2024-09-21T00:15:35.562 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:15:35] ENGINE Bus STARTED 2024-09-21T00:15:35.660 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.481+0000 7fd98d66e040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:15:36.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.658+0000 7fd98d66e040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:15:36.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:35.809+0000 7fd98d66e040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: mgrmap e37: y(active, starting, since 1.00071s), standbys: x 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:15:36.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: Manager daemon y is now available 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:15:36.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: mgrmap e37: y(active, starting, since 1.00071s), standbys: x 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:15:36.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: Manager daemon y is now available 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:36 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:15:36.379 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:36.168+0000 7fd98d66e040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:36.233+0000 7fd98d66e040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: mgrmap e37: y(active, starting, since 1.00071s), standbys: x 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:15:36.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: Manager daemon y is now available 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:15:36.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:15:36.753 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:36.435+0000 7fd98d66e040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:15:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:37 smithi031 ceph-mon[24497]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T00:15:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:37 smithi031 ceph-mon[24497]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:37.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-mon[23740]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-mon[23740]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-mon[28324]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-mon[28324]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:15:37.178+0000 7fe6e8151640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:00:15:37] ENGINE Error in HTTPServer.serve 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:15:37.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._connections.run(self.expiration_interval) 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._run(expiration_interval) 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s = self.context.wrap_socket( 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.sslsocket_class._create( 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self.do_handshake() 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:15:37.426 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._sslobj.do_handshake() 2024-09-21T00:15:37.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:15:37.427 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:15:37 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:15:37.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:37.406+0000 7fd98d66e040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:15:37.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:37.485+0000 7fd98d66e040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: mgrmap e39: y(active, since 3s), standbys: x 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: [21/Sep/2024:00:15:37] ENGINE Bus STARTING 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: [21/Sep/2024:00:15:37] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: [21/Sep/2024:00:15:37] ENGINE Error in HTTPServer.serve 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:15:38.170 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._run(expiration_interval) 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: s = self.context.wrap_socket( 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.sslsocket_class._create( 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.do_handshake() 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._sslobj.do_handshake() 2024-09-21T00:15:38.171 INFO:journalctl@ceph.mon.b.smithi031.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:15:38.172 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: [21/Sep/2024:00:15:37] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:15:38.172 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-mon[24497]: [21/Sep/2024:00:15:37] ENGINE Bus STARTED 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: mgrmap e39: y(active, since 3s), standbys: x 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: [21/Sep/2024:00:15:37] ENGINE Bus STARTING 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: [21/Sep/2024:00:15:37] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: [21/Sep/2024:00:15:37] ENGINE Error in HTTPServer.serve 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:15:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.do_handshake() 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: [21/Sep/2024:00:15:37] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:15:38.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[23740]: [21/Sep/2024:00:15:37] ENGINE Bus STARTED 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: mgrmap e39: y(active, since 3s), standbys: x 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: [21/Sep/2024:00:15:37] ENGINE Bus STARTING 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: [21/Sep/2024:00:15:37] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: [21/Sep/2024:00:15:37] ENGINE Error in HTTPServer.serve 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:15:38.177 INFO:journalctl@ceph.mon.c.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.do_handshake() 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: [21/Sep/2024:00:15:37] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:15:38.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:38 smithi018 ceph-mon[28324]: [21/Sep/2024:00:15:37] ENGINE Bus STARTED 2024-09-21T00:15:38.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.265+0000 7fd98d66e040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:15:38.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.386+0000 7fd98d66e040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:15:38.797 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.536+0000 7fd98d66e040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:15:38.798 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.609+0000 7fd98d66e040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:15:38.798 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.768+0000 7fd98d66e040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:15:39.079 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.843+0000 7fd98d66e040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:15:39.079 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:38.906+0000 7fd98d66e040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:15:39.079 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.078+0000 7fd98d66e040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:15:39.375 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.146+0000 7fd98d66e040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:15:39.375 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.217+0000 7fd98d66e040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:15:39.375 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.311+0000 7fd98d66e040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:15:39.375 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.373+0000 7fd98d66e040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:15:39.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:39.435+0000 7fd98d66e040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:15:40.301 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:40.296+0000 7fd98d66e040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:15:40.301 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:15:40] ENGINE Bus STARTING 2024-09-21T00:15:40.301 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: CherryPy Checker: 2024-09-21T00:15:40.301 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: The Application mounted at '' has an empty config. 2024-09-21T00:15:40.301 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:40.302 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-mon[24497]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:40.302 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-mon[24497]: mgrmap e40: y(active, since 5s), standbys: x 2024-09-21T00:15:40.302 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.302 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[23740]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[23740]: mgrmap e40: y(active, since 5s), standbys: x 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[28324]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[28324]: mgrmap e40: y(active, since 5s), standbys: x 2024-09-21T00:15:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:40 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:40.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: 2024-09-21T00:15:40.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:15:40] ENGINE Serving on http://:::9283 2024-09-21T00:15:40.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: [21/Sep/2024:00:15:40] ENGINE Bus STARTED 2024-09-21T00:15:41.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: Standby manager daemon x restarted 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:41 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: Standby manager daemon x restarted 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/2365423190' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:41 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: mgrmap e41: y(active, since 7s), standbys: x 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[23740]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: mgrmap e41: y(active, since 7s), standbys: x 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:42.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:42 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: mgrmap e41: y(active, since 7s), standbys: x 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:15:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:42 smithi031 ceph-mon[24497]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:15:43.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.320 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-21T00:15:43.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:15:43.321 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-mon[24497]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:15:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[23740]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T00:15:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:43 smithi018 ceph-mon[28324]: Reconfiguring daemon prometheus.a on smithi031 2024-09-21T00:15:43.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[44195]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:15:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T00:15:44.231 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:44 smithi031 ceph-mon[24497]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:15:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:44 smithi018 ceph-mon[23740]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:15:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:44 smithi018 ceph-mon[28324]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:15:45.368 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-mon[24497]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T00:15:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:45 smithi018 ceph-mon[23740]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T00:15:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:45 smithi018 ceph-mon[28324]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T00:15:45.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 systemd[1]: Stopping Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:15:46.097 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T00:15:46.098 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.781Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.782Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.782Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[56691]: ts=2024-09-21T00:15:45.782Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:45 smithi031 podman[59420]: 2024-09-21 00:15:45.791727411 +0000 UTC m=+0.122129161 container died a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:46.099 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59420]: 2024-09-21 00:15:46.096824967 +0000 UTC m=+0.427226756 container cleanup a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:46.100 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 bash[59420]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-21T00:15:46.442 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59432]: 2024-09-21 00:15:46.323019445 +0000 UTC m=+0.529064912 container remove a129a7d438ea9c91c16994602df2a3f7bd261f6607bbd0f5140c5106b897712f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 systemd[1]: Stopped Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service: Consumed 1.785s CPU time. 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 systemd[1]: Starting Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59523]: 2024-09-21 00:15:46.568635147 +0000 UTC m=+0.017880316 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T00:15:46.751 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59523]: 2024-09-21 00:15:46.699033507 +0000 UTC m=+0.148278711 container create 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:47.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59523]: 2024-09-21 00:15:46.934627341 +0000 UTC m=+0.383872496 container init 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:47.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 podman[59523]: 2024-09-21 00:15:46.93761665 +0000 UTC m=+0.386861801 container start 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:15:47.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.960Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T00:15:47.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.961Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T00:15:47.252 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.961Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi031 (none))" 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.961Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.961Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.963Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.963Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.965Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.965Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.967Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T00:15:47.253 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.968Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.504584ms 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.968Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.974Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.987Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.997Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:46.999Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.002Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-09-21T00:15:47.254 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.002Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.002Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.054µs wal_replay_duration=33.223423ms wbl_replay_duration=116ns total_replay_duration=34.798832ms 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.005Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.005Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.005Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 bash[59523]: 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 systemd[1]: Started Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.027Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.72391ms db_storage=921ns remote_storage=1.121µs web_handler=346ns query_engine=797ns scrape=295.221µs scrape_sd=89.087µs notify=15.764µs notify_sd=20.616µs rules=20.879721ms tracing=8.263µs 2024-09-21T00:15:47.255 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.027Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T00:15:47.256 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:15:47.027Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T00:15:48.099 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-mon[24497]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:15:48.099 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.100 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.100 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:48.100 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:47 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:48.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[23740]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:15:48.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[28324]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:48.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:47 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:15:49.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[28324]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:49.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:15:49.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:15:49.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:49.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:49.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:15:49.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:15:49.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:48 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:49.170 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:48 smithi031 ceph-mon[24497]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T00:15:49.171 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:48 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:15:49.171 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:48 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:15:49.171 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:48 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:15:49.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[28324]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[28324]: Upgrade: Updating mgr.x 2024-09-21T00:15:49.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[28324]: Deploying daemon mgr.x on smithi031 2024-09-21T00:15:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[23740]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[23740]: Upgrade: Updating mgr.x 2024-09-21T00:15:49.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:49 smithi018 ceph-mon[23740]: Deploying daemon mgr.x on smithi031 2024-09-21T00:15:49.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:49 smithi031 ceph-mon[24497]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:49.975 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:49 smithi031 ceph-mon[24497]: Upgrade: Updating mgr.x 2024-09-21T00:15:49.976 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:49 smithi031 ceph-mon[24497]: Deploying daemon mgr.x on smithi031 2024-09-21T00:15:50.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:50 smithi031 systemd[1]: Stopping Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:15:50.746 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:50 smithi031 podman[59869]: 2024-09-21 00:15:50.45904612 +0000 UTC m=+0.138504675 container died 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:15:51.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:50 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:51.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:50 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:51.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:50 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:51.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:50 smithi031 podman[59869]: 2024-09-21 00:15:50.845077709 +0000 UTC m=+0.524536262 container cleanup 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T00:15:51.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:50 smithi031 bash[59869]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:50 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:15:51.447 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 podman[59880]: 2024-09-21 00:15:51.096350295 +0000 UTC m=+0.632513761 container remove 1c08857ae5b728a22117250c880ab6094b7342354d5d342e7df52b37d260669a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, GIT_CLEAN=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T00:15:51.447 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:15:51.447 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T00:15:51.448 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 systemd[1]: Stopped Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:15:51.448 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service: Consumed 35.708s CPU time. 2024-09-21T00:15:51.706 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 systemd[1]: Starting Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:15:51.706 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 podman[60008]: 2024-09-21 00:15:51.561615307 +0000 UTC m=+0.016511638 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:15:51.963 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:51 smithi031 ceph-mon[24497]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:51.963 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 podman[60008]: 2024-09-21 00:15:51.706149009 +0000 UTC m=+0.161045339 container create e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70) 2024-09-21T00:15:51.963 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 podman[60008]: 2024-09-21 00:15:51.933730049 +0000 UTC m=+0.388626386 container init e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:15:51.963 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 podman[60008]: 2024-09-21 00:15:51.937967804 +0000 UTC m=+0.392864135 container start e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T00:15:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:51 smithi018 ceph-mon[23740]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:51 smithi018 ceph-mon[28324]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:52.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:51 smithi031 ceph-mgr[60027]: -- 172.21.15.31:0/3634109237 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56121178d0e0 con 0x56121176b400 2024-09-21T00:15:52.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 bash[60008]: e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 2024-09-21T00:15:52.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 systemd[1]: Started Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:15:52.251 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.069+0000 7fc7c877a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:15:52.252 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.134+0000 7fc7c877a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:15:52.550 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.295+0000 7fc7c877a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:15:52.550 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.368+0000 7fc7c877a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:15:52.550 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.548+0000 7fc7c877a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:15:52.823 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:52.697+0000 7fc7c877a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:15:53.101 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:53.035+0000 7fc7c877a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[28324]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[23740]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:53 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.493 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-mon[24497]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T00:15:53.493 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.494 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:53.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:53.099+0000 7fc7c877a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:15:53.494 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:53.320+0000 7fc7c877a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:15:54.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:54.339+0000 7fc7c877a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:15:54.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:54.410+0000 7fc7c877a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:15:55.391 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-mon[24497]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:55.391 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.213+0000 7fc7c877a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:15:55.391 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.335+0000 7fc7c877a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:15:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:55 smithi018 ceph-mon[23740]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:55 smithi018 ceph-mon[28324]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:55.730 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.486+0000 7fc7c877a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:15:55.730 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.550+0000 7fc7c877a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:15:55.731 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.729+0000 7fc7c877a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:15:56.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.800+0000 7fc7c877a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:15:56.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:55.864+0000 7fc7c877a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:15:56.335 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.034+0000 7fc7c877a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:15:56.335 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.108+0000 7fc7c877a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:15:56.335 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.180+0000 7fc7c877a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:15:56.335 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.269+0000 7fc7c877a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:15:56.335 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.333+0000 7fc7c877a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:15:56.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:56.398+0000 7fc7c877a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:15:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[23740]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[28324]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:57.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:57 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.456 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-mon[24497]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:15:57.456 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.456 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:57.264+0000 7fc7c877a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:15:57] ENGINE Bus STARTING 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: CherryPy Checker: 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: The Application mounted at '' has an empty config. 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:15:57] ENGINE Serving on http://:::9283 2024-09-21T00:15:57.457 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:15:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:15:57] ENGINE Bus STARTED 2024-09-21T00:15:58.168 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: Standby manager daemon x restarted 2024-09-21T00:15:58.169 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: Standby manager daemon x started 2024-09-21T00:15:58.169 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:58.169 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:58.169 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:58.169 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:58 smithi031 ceph-mon[24497]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: Standby manager daemon x restarted 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: Standby manager daemon x started 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[23740]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:58.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-21T00:15:58.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:15:58.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:15:58.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:15:58.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:15:58.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:58 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/771543955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:15:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:59 smithi031 ceph-mon[24497]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:15:59 smithi031 ceph-mon[24497]: mgrmap e42: y(active, since 24s), standbys: x 2024-09-21T00:15:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:59 smithi018 ceph-mon[23740]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:15:59 smithi018 ceph-mon[23740]: mgrmap e42: y(active, since 24s), standbys: x 2024-09-21T00:15:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:59 smithi018 ceph-mon[28324]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:15:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:15:59 smithi018 ceph-mon[28324]: mgrmap e42: y(active, since 24s), standbys: x 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:00.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:00.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.430 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:00.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:02.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:01] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all mgr 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all crash 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all mds 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:16:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all nfs 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all mgr 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all crash 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all mds 2024-09-21T00:16:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all nfs 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:16:02.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:01 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all mgr 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all crash 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all mds 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:16:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all nfs 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:16:02.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:01 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:16:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:02 smithi031 ceph-mon[24497]: Upgrade: Updating grafana.a 2024-09-21T00:16:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:02 smithi031 ceph-mon[24497]: Deploying daemon grafana.a on smithi031 2024-09-21T00:16:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:02 smithi018 ceph-mon[23740]: Upgrade: Updating grafana.a 2024-09-21T00:16:03.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:02 smithi018 ceph-mon[23740]: Deploying daemon grafana.a on smithi031 2024-09-21T00:16:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:02 smithi018 ceph-mon[28324]: Upgrade: Updating grafana.a 2024-09-21T00:16:03.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:02 smithi018 ceph-mon[28324]: Deploying daemon grafana.a on smithi031 2024-09-21T00:16:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:03 smithi018 ceph-mon[23740]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:04.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:03 smithi018 ceph-mon[28324]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:03 smithi031 ceph-mon[24497]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:05.297 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (13m) 25s ago 21m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (12m) 9s ago 21m 52.3M - dad864ee21e9 0244fcc6c1bb 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 unknown 25s ago 20m 54.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (14s) 9s ago 25m 69.2M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (10m) 25s ago 27m 474M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:16:05.983 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (27m) 25s ago 27m 74.1M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (26m) 9s ago 26m 55.1M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (26m) 25s ago 26m 55.4M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (13m) 25s ago 22m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (12m) 9s ago 21m 16.7M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (25m) 25s ago 25m 55.6M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (25m) 25s ago 25m 55.8M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (24m) 25s ago 24m 52.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (24m) 25s ago 24m 54.4M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:16:05.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (24m) 9s ago 24m 57.1M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (23m) 9s ago 23m 51.9M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (23m) 9s ago 23m 53.4M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (22m) 9s ago 22m 56.0M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (19s) 9s ago 22m 44.1M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (20m) 25s ago 20m 78.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:16:05.985 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (20m) 9s ago 20m 79.6M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:16:06.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:05 smithi018 ceph-mon[23740]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:05 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:05 smithi018 ceph-mon[28324]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:05 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:05 smithi031 ceph-mon[24497]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:05 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:06.396 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:16:06.396 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:16:06.397 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:16:06.398 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:16:06.398 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:16:06.398 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:16:06.398 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T00:16:06.750 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr" 2024-09-21T00:16:06.751 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T00:16:06.751 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T00:16:06.751 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T00:16:06.751 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:16:06.751 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:16:07.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[23740]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[23740]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/279124743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[28324]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[28324]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:06 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/279124743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:06 smithi031 ceph-mon[24497]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:06 smithi031 ceph-mon[24497]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:07.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:06 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/279124743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[23740]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s 2024-09-21T00:16:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[23740]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[23740]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[28324]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s 2024-09-21T00:16:08.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[28324]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:08.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:07 smithi018 ceph-mon[28324]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:07 smithi031 ceph-mon[24497]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s 2024-09-21T00:16:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:07 smithi031 ceph-mon[24497]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:08.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:07 smithi031 ceph-mon[24497]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:10.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:09 smithi018 ceph-mon[23740]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:09 smithi018 ceph-mon[28324]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:09 smithi031 ceph-mon[24497]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:16:11.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:10 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:11.154 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:10 smithi031 systemd[1]: Stopping Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:16:11.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:10 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:10 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:11.426 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[48841]: t=2024-09-21T00:16:11+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-21T00:16:11.426 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:11 smithi031 podman[61578]: 2024-09-21 00:16:11.161176659 +0000 UTC m=+0.121761595 container died 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, io.openshift.expose-services=, name=ubi8, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, release=236.1648460182, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, version=8.5) 2024-09-21T00:16:11.751 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:11 smithi031 podman[61578]: 2024-09-21 00:16:11.524845047 +0000 UTC m=+0.485429983 container cleanup 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vcs-type=git, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , release=236.1648460182, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5) 2024-09-21T00:16:11.751 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:11 smithi031 bash[61578]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a 2024-09-21T00:16:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:11 smithi018 ceph-mon[23740]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:11 smithi018 ceph-mon[28324]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:11] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.43.0" 2024-09-21T00:16:12.227 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:11 smithi031 ceph-mon[24497]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:12.227 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:11 smithi031 podman[61589]: 2024-09-21 00:16:11.868372868 +0000 UTC m=+0.707872792 container remove 0244fcc6c1bb9ba408209618630aae105c03cdddc8814aef5a0a7d78e69a54eb (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, version=8.5, release=236.1648460182, distribution-scope=public, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.24.2, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64) 2024-09-21T00:16:12.228 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T00:16:12.228 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 systemd[1]: Stopped Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:16:12.228 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Consumed 4.629s CPU time. 2024-09-21T00:16:12.228 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 systemd[1]: Starting Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:16:12.501 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 podman[61723]: 2024-09-21 00:16:12.340883665 +0000 UTC m=+0.017097451 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-09-21T00:16:12.501 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 podman[61723]: 2024-09-21 00:16:12.453266038 +0000 UTC m=+0.129479814 container create 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:16:12.815 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 podman[61723]: 2024-09-21 00:16:12.65556393 +0000 UTC m=+0.331777714 container init 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:16:12.815 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 podman[61723]: 2024-09-21 00:16:12.658560993 +0000 UTC m=+0.334774769 container start 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:16:12.815 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 bash[61723]: 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba 2024-09-21T00:16:12.815 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 systemd[1]: Started Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814600444Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814791251Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814807089Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814813311Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814817844Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814822462Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814827392Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814831964Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-21T00:16:13.076 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814836882Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.81484283Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814847849Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.81485236Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814877628Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.81488523Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814890581Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814895017Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814900839Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T00:16:13.077 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.814905888Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=settings t=2024-09-21T00:16:12.81491068Z level=info msg="App mode production" 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore t=2024-09-21T00:16:12.814972997Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore t=2024-09-21T00:16:12.814991492Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.827100763Z level=info msg="Starting DB migrations" 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.830501159Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.842100946Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T00:16:13.078 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.843293727Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.844062452Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.845245843Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.846377004Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.847667659Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.848308959Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.848802276Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.849344755Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.850115705Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.851229943Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.851563304Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.852648886Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.853734191Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T00:16:13.079 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.854811103Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.855159579Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.856236621Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.857315799Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.857624264Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.858710471Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.859782576Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.860235181Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.861018897Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.861794112Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.864119624Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.866770115Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.869210556Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.871990533Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T00:16:13.080 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.872071341Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.874733886Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.877426439Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.880164256Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.882735066Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.885335896Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.891675796Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.894132113Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.896544961Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.902909084Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.909293947Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.911600211Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.913992714Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.916440247Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T00:16:13.081 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.918924606Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.921258051Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.923700555Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.925999256Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.928489456Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.930907633Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.93332011Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.934084243Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.93524082Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.936058629Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.936850462Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.937670354Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.938461514Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T00:16:13.082 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.939631117Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.940417885Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.941257274Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.941339779Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.942041666Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.942123295Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.942813867Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.943586545Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.944380895Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.945158979Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.945941001Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.946709459Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.947516164Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T00:16:13.083 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.948312604Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.949084682Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.949852758Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.950608222Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.951457981Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.955299014Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.956456797Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.957617144Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.958347665Z level=info msg="Executing migration" id="add share column" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.959560657Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.960320741Z level=info msg="Executing migration" id="create file table" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.961090274Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.961847249Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.96263455Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T00:16:13.084 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.963348364Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.964063807Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.964839426Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.96603365Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.968632138Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.971334171Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.973923013Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T00:16:13.085 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.9765336Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.979219208Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.981882193Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.984656718Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.987193975Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.989923526Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.992629964Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:12 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:12.995089764Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.008475714Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.011184522Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.013873537Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.018647652Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T00:16:13.086 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.021130093Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.023874438Z level=info msg="Executing migration" id="create folder table" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.026465927Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.028883432Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.031257548Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.0336647Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=migrator t=2024-09-21T00:16:13.036239855Z level=info msg="migrations completed" performed=105 skipped=376 duration=205.764592ms 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore t=2024-09-21T00:16:13.036679453Z level=info msg="Created default organization" 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=plugin.finder t=2024-09-21T00:16:13.075000798Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=secrets t=2024-09-21T00:16:13.075093627Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T00:16:13.087 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=query_data t=2024-09-21T00:16:13.078820945Z level=info msg="Query Service initialization" 2024-09-21T00:16:13.364 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=live.push_http t=2024-09-21T00:16:13.083903881Z level=info msg="Live Push Gateway initialization" 2024-09-21T00:16:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-mon[24497]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:14.001 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=infra.usagestats.collector t=2024-09-21T00:16:13.736061931Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=provisioning.datasources t=2024-09-21T00:16:13.738009624Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=provisioning.datasources t=2024-09-21T00:16:13.738018371Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=provisioning.datasources t=2024-09-21T00:16:13.738123159Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=provisioning.alerting t=2024-09-21T00:16:13.760689531Z level=info msg="starting to provision alerting" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=provisioning.alerting t=2024-09-21T00:16:13.760700042Z level=info msg="finished to provision alerting" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=grafanaStorageLogger t=2024-09-21T00:16:13.760854501Z level=info msg="storage starting" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=ngalert.state.manager t=2024-09-21T00:16:13.760996402Z level=info msg="Warming state cache for startup" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=http.server t=2024-09-21T00:16:13.763870573Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=ngalert.state.manager t=2024-09-21T00:16:13.774102625Z level=info msg="State cache has been initialized" states=0 duration=13.104054ms 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=ngalert.multiorg.alertmanager t=2024-09-21T00:16:13.774160169Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=ticker t=2024-09-21T00:16:13.774230294Z level=info msg=starting first_tick=2024-09-21T00:16:20Z 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore.transactions t=2024-09-21T00:16:13.787888191Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T00:16:14.002 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore.transactions t=2024-09-21T00:16:13.799242136Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-21T00:16:14.003 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore.transactions t=2024-09-21T00:16:13.809827757Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-09-21T00:16:14.003 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:16:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=sqlstore.transactions t=2024-09-21T00:16:13.820474544Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=3 code="database is locked" 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[28324]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[23740]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:16.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:15 smithi031 ceph-mon[24497]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:16.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:15 smithi018 ceph-mon[23740]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:15 smithi018 ceph-mon[28324]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[23740]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[28324]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:17 smithi031 ceph-mon[24497]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:17 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:17 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:17 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:17 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:19 smithi018 ceph-mon[23740]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:19 smithi018 ceph-mon[28324]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:19 smithi031 ceph-mon[24497]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:21.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:20 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:20 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:20 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:20 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:20 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:20 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:21] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.43.0" 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:16:22.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:16:22.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:16:22.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:16:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:21 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:16:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:16:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:16:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:21 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[23740]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[23740]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[23740]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[23740]: Upgrade: Finalizing container_image settings 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[23740]: Upgrade: Complete! 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[28324]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[28324]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-09-21T00:16:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[28324]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[28324]: Upgrade: Finalizing container_image settings 2024-09-21T00:16:23.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:22 smithi018 ceph-mon[28324]: Upgrade: Complete! 2024-09-21T00:16:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:22 smithi031 ceph-mon[24497]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-09-21T00:16:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:22 smithi031 ceph-mon[24497]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-09-21T00:16:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:22 smithi031 ceph-mon[24497]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T00:16:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:22 smithi031 ceph-mon[24497]: Upgrade: Finalizing container_image settings 2024-09-21T00:16:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:22 smithi031 ceph-mon[24497]: Upgrade: Complete! 2024-09-21T00:16:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:24 smithi018 ceph-mon[23740]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:24 smithi018 ceph-mon[28324]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:25.233 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:24 smithi031 ceph-mon[24497]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:26 smithi018 ceph-mon[23740]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:26 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:26 smithi018 ceph-mon[28324]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:26 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:26 smithi031 ceph-mon[24497]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:26 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:28 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:16:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:28 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[23740]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[23740]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[23740]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[23740]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[28324]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[28324]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[28324]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[28324]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-21T00:16:29.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:29 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:29 smithi031 ceph-mon[24497]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:29 smithi031 ceph-mon[24497]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:29 smithi031 ceph-mon[24497]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:16:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:29 smithi031 ceph-mon[24497]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-09-21T00:16:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:29 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:16:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:30 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:30.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:30 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:30.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:30 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:32.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:31 smithi031 ceph-mon[24497]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:31 smithi018 ceph-mon[23740]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:31 smithi018 ceph-mon[28324]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:31] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.43.0" 2024-09-21T00:16:34.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:33 smithi018 ceph-mon[23740]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:34.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:33 smithi018 ceph-mon[28324]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:33 smithi031 ceph-mon[24497]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-09-21T00:16:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:35 smithi018 ceph-mon[23740]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-09-21T00:16:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:35 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:35 smithi018 ceph-mon[28324]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-09-21T00:16:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:35 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:35 smithi031 ceph-mon[24497]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 0 op/s 2024-09-21T00:16:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:35 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:37.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-21T00:16:38.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:37 smithi018 ceph-mon[23740]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:38.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:37 smithi018 ceph-mon[28324]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:38.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:37 smithi031 ceph-mon[24497]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:39.105 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:38 smithi018 ceph-mon[28324]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:39.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:38 smithi018 ceph-mon[23740]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:38 smithi031 ceph-mon[24497]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:40.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:39 smithi018 ceph-mon[23740]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:40.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:39 smithi018 ceph-mon[28324]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:40.011 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:16:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:39 smithi031 ceph-mon[24497]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:16:40.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-09-21T00:16:41.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:40 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:41.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:40 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3042604387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:41.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:40 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:41.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:40 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3042604387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:40 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:40 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3042604387' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:42.166 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:41 smithi018 ceph-mon[23740]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:42.167 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:41] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.43.0" 2024-09-21T00:16:42.167 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:41 smithi018 ceph-mon[28324]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:41 smithi031 ceph-mon[24497]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:42.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-21T00:16:43.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:42 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/1111775046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:42 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/1111775046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:42 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/1111775046' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:43.467 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-09-21T00:16:44.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:43 smithi018 ceph-mon[23740]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:44.071 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:43 smithi018 ceph-mon[28324]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:43 smithi031 ceph-mon[24497]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:45.624 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:16:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:45 smithi018 ceph-mon[28324]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2737360179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:45 smithi018 ceph-mon[23740]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2737360179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:46.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-09-21T00:16:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:45 smithi031 ceph-mon[24497]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:45 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2737360179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:16:48.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:47 smithi018 ceph-mon[23740]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:48.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:47 smithi018 ceph-mon[28324]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:47 smithi031 ceph-mon[24497]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:49.586 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:16:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:49 smithi018 ceph-mon[23740]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:49 smithi018 ceph-mon[23740]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:49 smithi018 ceph-mon[28324]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:49 smithi018 ceph-mon[28324]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:50.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:16:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:49 smithi031 ceph-mon[24497]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:49 smithi031 ceph-mon[24497]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:51.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:50 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:50 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:50 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:50 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:50 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:16:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:50 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:16:52.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:51 smithi018 ceph-mon[23740]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:51 smithi018 ceph-mon[28324]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:16:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:16:51] "GET /metrics HTTP/1.1" 200 37979 "" "Prometheus/2.43.0" 2024-09-21T00:16:52.238 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:16:52.238 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:16:52.238 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:16:52.239 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:16:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:51 smithi031 ceph-mon[24497]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:52.818 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:16:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:53 smithi018 ceph-mon[28324]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:53 smithi018 ceph-mon[28324]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:54.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:53 smithi018 ceph-mon[23740]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:54.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:53 smithi018 ceph-mon[23740]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:53 smithi031 ceph-mon[24497]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:53 smithi031 ceph-mon[24497]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:54.955 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:16:54.956 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:16:54.956 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:16:55.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-09-21T00:16:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:55 smithi018 ceph-mon[23740]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:56.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/2261369911' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:16:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:55 smithi018 ceph-mon[28324]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:56.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/2261369911' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:16:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:55 smithi031 ceph-mon[24497]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:16:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:55 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/2261369911' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:16:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:57 smithi018 ceph-mon[23740]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:58.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:57 smithi018 ceph-mon[28324]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:58.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:57 smithi031 ceph-mon[24497]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:16:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:58 smithi018 ceph-mon[23740]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:59.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:58 smithi018 ceph-mon[28324]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:59.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:58 smithi031 ceph-mon[24497]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:16:59.517 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:00.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[23740]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi031", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:00.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[23740]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:00.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:00.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[28324]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi031", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:00.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[28324]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:00.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:16:59 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:00.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:17:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:59 smithi031 ceph-mon[24497]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi031", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:59 smithi031 ceph-mon[24497]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:16:59 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[23740]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[28324]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:01.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:00 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:00 smithi031 ceph-mon[24497]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:00 smithi031 ceph-mon[24497]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:01.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:01.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:01.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:00 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:01] "GET /metrics HTTP/1.1" 200 37979 "" "Prometheus/2.43.0" 2024-09-21T00:17:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[28324]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[28324]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-21T00:17:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[28324]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[28324]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[23740]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[23740]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-21T00:17:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[23740]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:02.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:01 smithi018 ceph-mon[23740]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:01 smithi031 ceph-mon[24497]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:01 smithi031 ceph-mon[24497]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-21T00:17:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:01 smithi031 ceph-mon[24497]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:01 smithi031 ceph-mon[24497]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:02.472 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (14m) 82s ago 22m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (50s) 35s ago 22m 74.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 unknown 82s ago 21m 54.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (71s) 35s ago 26m 439M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (11m) 82s ago 28m 474M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (28m) 82s ago 28m 74.1M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:17:03.168 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (27m) 35s ago 27m 56.7M 2048M 17.2.0 e1d6a67b021e c3cbca6cde44 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (27m) 82s ago 27m 55.4M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (14m) 82s ago 22m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (13m) 35s ago 22m 16.8M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (26m) 82s ago 26m 55.6M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (26m) 82s ago 26m 55.8M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (25m) 82s ago 25m 52.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (25m) 82s ago 25m 54.4M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (24m) 35s ago 24m 57.1M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (24m) 35s ago 24m 52.0M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (24m) 35s ago 24m 53.5M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (23m) 35s ago 23m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (76s) 35s ago 23m 54.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:17:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (21m) 82s ago 21m 78.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:17:03.170 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (21m) 35s ago 21m 79.7M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:17:03.583 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:17:03.584 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:17:03.931 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi031", 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/1 daemons upgraded", 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T00:17:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:17:03.933 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:17:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T00:17:04.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:03 smithi031 ceph-mon[24497]: from='client.? 172.21.15.18:0/3089911102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:04.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3089911102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T00:17:04.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:03 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3089911102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:05.127 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:05.127 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:05.127 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:05.127 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: Upgrade: It appears safe to stop mon.b 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:05.128 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:04 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: Upgrade: It appears safe to stop mon.b 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: Upgrade: It appears safe to stop mon.b 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:05.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:05.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:05.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:04 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:06.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:05 smithi031 ceph-mon[24497]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:06.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:05 smithi031 ceph-mon[24497]: Upgrade: Updating mon.b 2024-09-21T00:17:06.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:05 smithi031 ceph-mon[24497]: Deploying daemon mon.b on smithi031 2024-09-21T00:17:06.086 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:05 smithi031 ceph-mon[24497]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:06.344 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 systemd[1]: Stopping Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[23740]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[23740]: Upgrade: Updating mon.b 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[23740]: Deploying daemon mon.b on smithi031 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[28324]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[28324]: Upgrade: Updating mon.b 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[28324]: Deploying daemon mon.b on smithi031 2024-09-21T00:17:06.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:05 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:06.677 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b[24493]: 2024-09-21T00:17:06.342+0000 7fd48c55a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:17:06.677 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b[24493]: 2024-09-21T00:17:06.342+0000 7fd48c55a700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T00:17:06.677 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 podman[63703]: 2024-09-21 00:17:06.424382906 +0000 UTC m=+0.189659044 container died c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., version=8, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-type=git, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-09-21T00:17:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 podman[63703]: 2024-09-21 00:17:06.776096312 +0000 UTC m=+0.541372439 container cleanup c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vcs-type=git, com.redhat.component=centos-stream-container, release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, architecture=x86_64) 2024-09-21T00:17:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:06 smithi031 bash[63703]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b 2024-09-21T00:17:07.460 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 podman[63717]: 2024-09-21 00:17:07.102388585 +0000 UTC m=+0.671177681 container remove c3cbca6cde4415e70cec66bf82b33c04630fb372ee1ea63f741492d9a399be9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-type=git, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., version=8, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public) 2024-09-21T00:17:07.460 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T00:17:07.460 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 systemd[1]: Stopped Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:17:07.460 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.b.service: Consumed 19.183s CPU time. 2024-09-21T00:17:07.460 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 systemd[1]: Starting Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:17:07.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 podman[63844]: 2024-09-21 00:17:07.669958874 +0000 UTC m=+0.110515446 container create e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:17:07.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 podman[63844]: 2024-09-21 00:17:07.575763885 +0000 UTC m=+0.016320468 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 podman[63844]: 2024-09-21 00:17:07.872812862 +0000 UTC m=+0.313369437 container init e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70) 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 podman[63844]: 2024-09-21 00:17:07.876040595 +0000 UTC m=+0.316597168 container start e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable), process ceph-mon, pid 2 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: pidfile_write: ignore empty --pid-file 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: load: jerasure load: lrc 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: RocksDB version: 7.9.2 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Git sha 0 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Compile date 2024-09-19 18:20:48 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: DB SUMMARY 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: DB Session ID: YKN3NN0XXNXE7MWACRTN 2024-09-21T00:17:08.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: CURRENT file: CURRENT 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 532853 ; 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.error_if_exists: 0 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.create_if_missing: 0 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.paranoid_checks: 1 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.env: 0x55b86c932940 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T00:17:08.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.info_log: 0x55b86dda16e0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.statistics: (nil) 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.use_fsync: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_log_file_size: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_fallocate: 1 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T00:17:08.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.use_direct_reads: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.db_log_dir: 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.wal_dir: 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T00:17:08.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.write_buffer_manager: 0x55b86ddb0dc0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.unordered_write: 0 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T00:17:08.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T00:17:08.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.row_cache: None 2024-09-21T00:17:08.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.wal_filter: None 2024-09-21T00:17:08.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.two_write_queues: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.wal_compression: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.atomic_flush: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T00:17:08.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.log_readahead_size: 0 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_background_jobs: 2 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_background_compactions: -1 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_subcompactions: 1 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T00:17:08.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_open_files: -1 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_background_flushes: -1 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Compression algorithms supported: 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kZSTD supported: 0 2024-09-21T00:17:08.260 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kXpressCompression supported: 0 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kBZip2Compression supported: 0 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kLZ4Compression supported: 1 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kZlibCompression supported: 1 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: kSnappyCompression supported: 1 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T00:17:08.261 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.merge_operator: 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_filter: None 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b86dda1320) 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: cache_index_and_filter_blocks: 1 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: pin_top_level_index_and_filter: 1 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_type: 0 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: data_block_index_type: 0 2024-09-21T00:17:08.262 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_shortening: 1 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: checksum: 4 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: no_block_cache: 0 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache: 0x55b86dd951f0 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_name: BinnedLRUCache 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_options: 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: capacity : 536870912 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: num_shard_bits : 4 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: strict_capacity_limit : 0 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: high_pri_pool_ratio: 0.000 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_cache_compressed: (nil) 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: persistent_cache: (nil) 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_size: 4096 2024-09-21T00:17:08.263 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_size_deviation: 10 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_restart_interval: 16 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: index_block_restart_interval: 1 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: metadata_block_size: 4096 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: partition_filters: 0 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: use_delta_encoding: 1 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: filter_policy: bloomfilter 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: whole_key_filtering: 1 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: verify_compression: 0 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: read_amp_bytes_per_bit: 0 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: format_version: 5 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: enable_index_compression: 1 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: block_align: 0 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: max_auto_readahead_size: 262144 2024-09-21T00:17:08.264 INFO:journalctl@ceph.mon.b.smithi031.stdout: prepopulate_block_cache: 0 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout: initial_auto_readahead_size: 8192 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression: NoCompression 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.num_levels: 7 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T00:17:08.265 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:17:08.266 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T00:17:08.267 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T00:17:08.268 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T00:17:08.269 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T00:17:08.270 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T00:17:08.271 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.inplace_update_support: 0 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T00:17:08.272 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.bloom_locality: 0 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.max_successive_merges: 0 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.ttl: 2592000 2024-09-21T00:17:08.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enable_blob_files: false 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.min_blob_size: 0 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T00:17:08.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T00:17:08.275 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28967, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d408432b-0f14-4b72-b64f-f5a3dec13046 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877827903411, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-09-21T00:17:08.276 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877827905578, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 346812, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28972, "largest_seqno": 29395, "table_properties": {"data_size": 344472, "index_size": 852, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 3868, "raw_average_key_size": 23, "raw_value_size": 341030, "raw_average_value_size": 2066, "num_data_blocks": 39, "num_entries": 165, "num_filter_entries": 165, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726877827, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d408432b-0f14-4b72-b64f-f5a3dec13046", "db_session_id": "YKN3NN0XXNXE7MWACRTN", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877827905680, "job": 1, "event": "recovery_finished"} 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b86ddc2e00 2024-09-21T00:17:08.277 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: DB pointer 0x55b86debe000 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** DB Stats ** 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:08.278 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-21T00:17:08.279 INFO:journalctl@ceph.mon.b.smithi031.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: L0 1/0 338.68 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 270.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: L6 1/0 9.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: Sum 2/0 9.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 270.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 270.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.280 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** Compaction Stats [default] ** 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 270.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T00:17:08.281 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T00:17:08.282 INFO:journalctl@ceph.mon.b.smithi031.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: Cumulative compaction: 0.00 GB write, 73.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: Interval compaction: 0.00 GB write, 73.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: Block cache BinnedLRUCache@0x55b86dd951f0#2 capacity: 512.00 MB usage: 1.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,0.95 KB,0.000181794%) Misc(1,0.00 KB,0%) 2024-09-21T00:17:08.283 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: starting mon.b rank 2 at public addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] at bind addrs [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???) e3 preinit fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).mds e1 new map 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).mds e1 print_map 2024-09-21T00:17:08.284 INFO:journalctl@ceph.mon.b.smithi031.stdout: e1 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: legacy client fscid: -1 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout: No filesystems configured 2024-09-21T00:17:08.285 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T00:17:08.286 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:08.286 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:08.286 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:08.286 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 ceph-mon[63863]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-09-21T00:17:08.286 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 bash[63844]: e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c 2024-09-21T00:17:08.287 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:07 smithi031 systemd[1]: Started Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:17:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: mon.b calling monitor election 2024-09-21T00:17:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: mon.c calling monitor election 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: mon.a calling monitor election 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: fsmap 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: mgrmap e42: y(active, since 94s), standbys: x 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[23740]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: mon.b calling monitor election 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: mon.c calling monitor election 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: mon.a calling monitor election 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: fsmap 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: mgrmap e42: y(active, since 94s), standbys: x 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:09.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:09.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:09 smithi018 ceph-mon[28324]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: mon.b calling monitor election 2024-09-21T00:17:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: mon.c calling monitor election 2024-09-21T00:17:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: mon.a calling monitor election 2024-09-21T00:17:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: fsmap 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: mgrmap e42: y(active, since 94s), standbys: x 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:09.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:09 smithi031 ceph-mon[63863]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:10.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:10 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:10.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:10 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:10 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:11 smithi018 ceph-mon[23740]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:11 smithi018 ceph-mon[28324]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:11] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.43.0" 2024-09-21T00:17:12.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:11 smithi031 ceph-mon[63863]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:13.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:12.916+0000 7fe71e3fa640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[28324]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.319 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[23740]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:13.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.593 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:13 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.594 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:13 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.594 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:13 smithi031 ceph-mon[63863]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:13.594 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:13 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.594 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:13 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:13.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:13 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:15.645 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:15 smithi031 ceph-mon[63863]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:15.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:15 smithi018 ceph-mon[23740]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:15 smithi018 ceph-mon[28324]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:17.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Detected new or changed devices on smithi031 2024-09-21T00:17:17.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all crash 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all mds 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:17:17.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all nfs 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Detected new or changed devices on smithi031 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all crash 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all mds 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all nfs 2024-09-21T00:17:17.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:17:17.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all grafana 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all loki 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all promtail 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Finalizing container_image settings 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:17:17.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: Upgrade: Complete! 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all grafana 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all loki 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all promtail 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Finalizing container_image settings 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:17:17.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:17:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: Upgrade: Complete! 2024-09-21T00:17:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-21T00:17:17.440 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:17 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Detected new or changed devices on smithi031 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mds 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nfs 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all grafana 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all loki 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all promtail 2024-09-21T00:17:17.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:17:17.506 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.507 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-21T00:17:17.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:17 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:19.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:19 smithi031 ceph-mon[63863]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-21T00:17:19.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:19 smithi018 ceph-mon[23740]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-21T00:17:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:19 smithi018 ceph-mon[28324]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:20.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:20 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:20 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:20 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:21 smithi018 ceph-mon[23740]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:21 smithi018 ceph-mon[28324]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:21] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-09-21T00:17:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:21 smithi031 ceph-mon[63863]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:23 smithi018 ceph-mon[23740]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:17:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:23 smithi018 ceph-mon[28324]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:17:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:23 smithi031 ceph-mon[63863]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:17:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:25 smithi018 ceph-mon[23740]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:17:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:25 smithi018 ceph-mon[28324]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:17:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:25 smithi031 ceph-mon[63863]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:17:28.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:27 smithi018 ceph-mon[23740]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:28.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:27 smithi018 ceph-mon[28324]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:27 smithi031 ceph-mon[63863]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:17:30.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:29 smithi018 ceph-mon[23740]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:30.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:29 smithi018 ceph-mon[28324]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:29 smithi031 ceph-mon[63863]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:31.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:30 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:31.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:30 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:30 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:31 smithi018 ceph-mon[23740]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:31 smithi018 ceph-mon[28324]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:31] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-09-21T00:17:32.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:31 smithi031 ceph-mon[63863]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:34.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:33 smithi018 ceph-mon[23740]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:34.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:33 smithi018 ceph-mon[28324]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:33 smithi031 ceph-mon[63863]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:35.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:17:36.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[23740]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:36.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[23740]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:36.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:36.049 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[28324]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:36.049 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[28324]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:36.049 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:35 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:35 smithi031 ceph-mon[63863]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:35 smithi031 ceph-mon[63863]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:35 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:37.161 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:17:37.161 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (14m) 116s ago 22m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:17:37.161 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (84s) 24s ago 22m 76.7M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:17:37.161 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 unknown 116s ago 21m 54.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (105s) 24s ago 27m 440M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (11m) 116s ago 28m 474M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (29m) 116s ago 29m 74.1M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (29s) 24s ago 27m 18.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (27m) 116s ago 27m 55.4M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (14m) 116s ago 23m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (14m) 24s ago 23m 16.5M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (27m) 116s ago 27m 55.6M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (26m) 116s ago 26m 55.8M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (26m) 116s ago 26m 52.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (25m) 116s ago 25m 54.4M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (25m) 24s ago 25m 57.4M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:17:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (25m) 24s ago 25m 52.1M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:17:37.163 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (24m) 24s ago 24m 53.5M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:17:37.163 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (24m) 24s ago 24m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:17:37.163 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (110s) 24s ago 23m 62.5M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:17:37.163 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (22m) 116s ago 22m 78.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:17:37.163 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (22m) 24s ago 22m 80.2M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:17:37.897 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-09-21T00:17:38.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:37 smithi018 ceph-mon[23740]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:38.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:37 smithi018 ceph-mon[28324]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:38.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:37 smithi031 ceph-mon[63863]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:38 smithi018 ceph-mon[23740]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:38 smithi018 ceph-mon[28324]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:38 smithi031 ceph-mon[63863]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:40.097 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:17:40.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:39 smithi018 ceph-mon[23740]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:40.098 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:39 smithi018 ceph-mon[28324]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:39 smithi031 ceph-mon[63863]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:40.719 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:17:41.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:40 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:41.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:40 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/210902889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:41.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:40 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:41.027 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:40 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/210902889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:40 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:40 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/210902889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:42.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:41 smithi018 ceph-mon[23740]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:42.167 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:41] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-09-21T00:17:42.168 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:41 smithi018 ceph-mon[28324]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:41 smithi031 ceph-mon[63863]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:42.661 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:17:42.662 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:17:43.241 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:17:44.166 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:43 smithi018 ceph-mon[28324]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:44.166 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:43 smithi018 ceph-mon[28324]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:44.166 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:43 smithi018 ceph-mon[23740]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:44.166 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:43 smithi018 ceph-mon[23740]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:43 smithi031 ceph-mon[63863]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:44.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:43 smithi031 ceph-mon[63863]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:45.332 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:45.332 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:45.332 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:45.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:45 smithi018 ceph-mon[23740]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:45.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:45 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/3007370237' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:17:45.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:45 smithi018 ceph-mon[28324]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:45.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:45 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/3007370237' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:17:45.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-09-21T00:17:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:45 smithi031 ceph-mon[63863]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:46.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:45 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3007370237' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:17:48.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:47 smithi018 ceph-mon[23740]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:48.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:47 smithi018 ceph-mon[28324]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:47 smithi031 ceph-mon[63863]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:17:49.734 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi018", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:50.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi018", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:50.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:49 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "mon", "hosts": "smithi018", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:17:50.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:49 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:50.351 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:50 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:17:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-09-21T00:17:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:51.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:50 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:17:52.062 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:17:51] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: Upgrade: It appears safe to stop mon.a 2024-09-21T00:17:52.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[23740]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: Upgrade: It appears safe to stop mon.a 2024-09-21T00:17:52.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:52 smithi018 ceph-mon[28324]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:53.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:53.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:17:53.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: Upgrade: It appears safe to stop mon.a 2024-09-21T00:17:53.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:52 smithi031 ceph-mon[63863]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:54.657 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[23740]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[23740]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[28324]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:55.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:55.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:54 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:54 smithi031 ceph-mon[63863]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:54 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' 2024-09-21T00:17:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:54 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:17:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:54 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:17:55.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:54 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (15m) 2m ago 23m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (102s) 43s ago 22m 76.7M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 unknown 2m ago 22m 54.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (2m) 43s ago 27m 440M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (11m) 2m ago 29m 474M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (29m) 2m ago 29m 74.1M 2048M 17.2.0 e1d6a67b021e 383c2536e0eb 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (47s) 43s ago 27m 18.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (28m) 2m ago 28m 55.4M 2048M 17.2.0 e1d6a67b021e b12e4f83e07a 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (15m) 2m ago 23m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (14m) 43s ago 23m 16.5M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:17:55.345 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (27m) 2m ago 27m 55.6M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (27m) 2m ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (26m) 2m ago 26m 52.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (26m) 2m ago 26m 54.4M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (25m) 43s ago 25m 57.4M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (25m) 43s ago 25m 52.1M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (25m) 43s ago 25m 53.5M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (24m) 43s ago 24m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (2m) 43s ago 23m 62.5M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (22m) 2m ago 22m 78.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:17:55.346 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (22m) 43s ago 22m 80.2M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:17:55.771 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:17:55.771 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:17:55.771 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T00:17:55.771 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-21T00:17:55.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": {}, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:17:55.772 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:17:56.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:55 smithi018 systemd[1]: Stopping Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:17:56.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[23740]: Upgrade: Updating mon.a 2024-09-21T00:17:56.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[23740]: Deploying daemon mon.a on smithi018 2024-09-21T00:17:56.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[23740]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:56.050 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[23740]: from='client.? 172.21.15.18:0/869344823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:56.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[28324]: Upgrade: Updating mon.a 2024-09-21T00:17:56.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[28324]: Deploying daemon mon.a on smithi018 2024-09-21T00:17:56.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[28324]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:56.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:55 smithi018 ceph-mon[28324]: from='client.? 172.21.15.18:0/869344823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi018", 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/2 daemons upgraded", 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T00:17:56.138 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:17:56.139 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:17:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:55 smithi031 ceph-mon[63863]: Upgrade: Updating mon.a 2024-09-21T00:17:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:55 smithi031 ceph-mon[63863]: Deploying daemon mon.a on smithi018 2024-09-21T00:17:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:55 smithi031 ceph-mon[63863]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:56.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:55 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/869344823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:17:56.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE Bus STOPPING 2024-09-21T00:17:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[23736]: 2024-09-21T00:17:56.047+0000 7f906bc82700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:17:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[23736]: 2024-09-21T00:17:56.047+0000 7f906bc82700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T00:17:56.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 podman[78052]: 2024-09-21 00:17:56.282777632 +0000 UTC m=+0.345208336 container died 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, release=754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, ceph=True, io.buildah.version=1.19.8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, version=8) 2024-09-21T00:17:56.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 podman[78052]: 2024-09-21 00:17:56.655759822 +0000 UTC m=+0.718190516 container cleanup 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-09-21T00:17:56.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 bash[78052]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a 2024-09-21T00:17:56.916 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:17:56.916 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE Bus STOPPED 2024-09-21T00:17:56.917 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE Bus STARTING 2024-09-21T00:17:56.917 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE Serving on http://:::9283 2024-09-21T00:17:56.917 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:56 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:17:56] ENGINE Bus STARTED 2024-09-21T00:17:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:56 smithi018 podman[78069]: 2024-09-21 00:17:56.915476473 +0000 UTC m=+0.625453049 container remove 383c2536e0ebd1cef29ed1826719dc94bf146e69d729db49f2a196599a53adda (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, name=centos-stream, ceph=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_BRANCH=HEAD, version=8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.buildah.version=1.19.8) 2024-09-21T00:17:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T00:17:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 systemd[1]: Stopped Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:17:57.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service: Consumed 28.149s CPU time. 2024-09-21T00:17:57.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 systemd[1]: Starting Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:17:57.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 podman[78208]: 2024-09-21 00:17:57.475209365 +0000 UTC m=+0.113132503 container create 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T00:17:57.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 podman[78208]: 2024-09-21 00:17:57.378957546 +0000 UTC m=+0.016880704 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 podman[78208]: 2024-09-21 00:17:57.711154624 +0000 UTC m=+0.349077782 container init 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 podman[78208]: 2024-09-21 00:17:57.71441565 +0000 UTC m=+0.352338796 container start 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable), process ceph-mon, pid 2 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: pidfile_write: ignore empty --pid-file 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: load: jerasure load: lrc 2024-09-21T00:17:58.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: RocksDB version: 7.9.2 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Git sha 0 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Compile date 2024-09-19 18:20:48 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: DB SUMMARY 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: DB Session ID: R27HF7B9ENKKDFY24XP5 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: CURRENT file: CURRENT 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2889 Bytes 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 1317806 ; 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.error_if_exists: 0 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.create_if_missing: 0 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.paranoid_checks: 1 2024-09-21T00:17:58.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.env: 0x55728fefa940 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.info_log: 0x557291e656e0 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T00:17:58.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.statistics: (nil) 2024-09-21T00:17:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.use_fsync: 0 2024-09-21T00:17:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_log_file_size: 0 2024-09-21T00:17:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T00:17:58.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_fallocate: 1 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.use_direct_reads: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.db_log_dir: 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.wal_dir: 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T00:17:58.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.write_buffer_manager: 0x557291e74dc0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T00:17:58.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.unordered_write: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.row_cache: None 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.wal_filter: None 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.two_write_queues: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.wal_compression: 0 2024-09-21T00:17:58.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.atomic_flush: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.log_readahead_size: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_background_jobs: 2 2024-09-21T00:17:58.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_background_compactions: -1 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_subcompactions: 1 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_open_files: -1 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T00:17:58.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_background_flushes: -1 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Compression algorithms supported: 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kZSTD supported: 0 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kXpressCompression supported: 0 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kBZip2Compression supported: 0 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kLZ4Compression supported: 1 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kZlibCompression supported: 1 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: kSnappyCompression supported: 1 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T00:17:58.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.merge_operator: 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_filter: None 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557291e65320) 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks: 1 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_top_level_index_and_filter: 1 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_type: 0 2024-09-21T00:17:58.185 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_index_type: 0 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_shortening: 1 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: checksum: 4 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: no_block_cache: 0 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache: 0x557291e591f0 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_name: BinnedLRUCache 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_options: 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: capacity : 536870912 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_shard_bits : 4 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: strict_capacity_limit : 0 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: high_pri_pool_ratio: 0.000 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_compressed: (nil) 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: persistent_cache: (nil) 2024-09-21T00:17:58.186 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size: 4096 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size_deviation: 10 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_restart_interval: 16 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_block_restart_interval: 1 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: metadata_block_size: 4096 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: partition_filters: 0 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: use_delta_encoding: 1 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: filter_policy: bloomfilter 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: whole_key_filtering: 1 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: verify_compression: 0 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: read_amp_bytes_per_bit: 0 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: format_version: 5 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_index_compression: 1 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_align: 0 2024-09-21T00:17:58.187 INFO:journalctl@ceph.mon.a.smithi018.stdout: max_auto_readahead_size: 262144 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout: prepopulate_block_cache: 0 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout: initial_auto_readahead_size: 8192 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression: NoCompression 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.num_levels: 7 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T00:17:58.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:17:58.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T00:17:58.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T00:17:58.191 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.inplace_update_support: 0 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T00:17:58.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.bloom_locality: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.max_successive_merges: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.ttl: 2592000 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enable_blob_files: false 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.min_blob_size: 0 2024-09-21T00:17:58.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26854, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a4fa1e9f-3b76-4262-bd89-0375c0435f77 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877877742516, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-09-21T00:17:58.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877877747969, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1236061, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26855, "largest_seqno": 28193, "table_properties": {"data_size": 1230635, "index_size": 2913, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 14736, "raw_average_key_size": 25, "raw_value_size": 1218621, "raw_average_value_size": 2083, "num_data_blocks": 132, "num_entries": 585, "num_filter_entries": 585, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726877877, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a4fa1e9f-3b76-4262-bd89-0375c0435f77", "db_session_id": "R27HF7B9ENKKDFY24XP5", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877877748090, "job": 1, "event": "recovery_finished"} 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557291e86e00 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: DB pointer 0x557291f82000 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** DB Stats ** 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:58.195 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 1/0 1.18 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: L6 1/0 9.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 10.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.196 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 383.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 147.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:17:58.197 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 147.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: Block cache BinnedLRUCache@0x557291e591f0#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: starting mon.a rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???) e3 preinit fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).mds e1 new map 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).mds e1 print_map 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: e1 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T00:17:58.198 INFO:journalctl@ceph.mon.a.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout: legacy client fscid: -1 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout: No filesystems configured 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 ceph-mon[78227]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 bash[78208]: 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 2024-09-21T00:17:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:57 smithi018 systemd[1]: Started Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ignoring --setuser ceph since I am not root 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ignoring --setgroup ceph since I am not root 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mgr[61431]: -- 172.21.15.18:0/3290751930 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562a722950e0 con 0x562a72273000 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.349+0000 7fd0b2a98040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:17:59.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: mon.a calling monitor election 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: monmap epoch 3 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: last_changed 2024-09-20T23:49:57.519417+0000 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: min_mon_release 17 (quincy) 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: election_strategy: 1 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:17:59.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: fsmap 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: from='mgr.15312 ' entity='' 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[78227]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:17:59.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: mon.a calling monitor election 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: monmap epoch 3 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: last_changed 2024-09-20T23:49:57.519417+0000 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: min_mon_release 17 (quincy) 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: election_strategy: 1 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:17:59.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: fsmap 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: from='mgr.15312 ' entity='' 2024-09-21T00:17:59.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-mon[28324]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-21T00:17:59.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: ignoring --setuser ceph since I am not root 2024-09-21T00:17:59.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: ignoring --setgroup ceph since I am not root 2024-09-21T00:17:59.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mgr[60027]: -- 172.21.15.31:0/1894104909 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562593b9d0e0 con 0x562593b7b400 2024-09-21T00:17:59.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.341+0000 7f78fea03040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.401+0000 7f78fea03040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: from='mgr.15312 172.21.15.18:0/1559135031' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: mon.a calling monitor election 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: monmap epoch 3 2024-09-21T00:17:59.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: last_changed 2024-09-20T23:49:57.519417+0000 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: min_mon_release 17 (quincy) 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: election_strategy: 1 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: fsmap 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in unknown state 2024-09-21T00:17:59.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: from='mgr.15312 ' entity='' 2024-09-21T00:17:59.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-mon[63863]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-21T00:17:59.635 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.409+0000 7fd0b2a98040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T00:17:59.636 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.563+0000 7fd0b2a98040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:17:59.636 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.633+0000 7fd0b2a98040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:17:59.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.791+0000 7fd0b2a98040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:17:59.929 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.556+0000 7f78fea03040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T00:17:59.929 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.626+0000 7f78fea03040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T00:17:59.929 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.779+0000 7f78fea03040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:17:59.929 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:17:59 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:17:59.926+0000 7f78fea03040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:18:00.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:17:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:17:59.938+0000 7fd0b2a98040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T00:18:00.516 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:00.257+0000 7f78fea03040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:18:00.517 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:00.321+0000 7f78fea03040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:18:00.535 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:00 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:00.271+0000 7fd0b2a98040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T00:18:00.535 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:00 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:00.337+0000 7fd0b2a98040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T00:18:00.535 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:00 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:00.533+0000 7fd0b2a98040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:18:01.001 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:00.514+0000 7f78fea03040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T00:18:01.448 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:01 smithi031 ceph-mon[63863]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-21T00:18:01.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:01 smithi018 ceph-mon[78227]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-21T00:18:01.470 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:01 smithi018 ceph-mon[28324]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-21T00:18:01.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:01.446+0000 7f78fea03040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:18:01.751 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:01.521+0000 7f78fea03040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:18:01.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:01.467+0000 7fd0b2a98040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T00:18:01.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:01.538+0000 7fd0b2a98040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T00:18:02.568 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.294+0000 7fd0b2a98040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:18:02.568 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.415+0000 7fd0b2a98040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:18:02.575 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.300+0000 7f78fea03040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T00:18:02.575 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.422+0000 7f78fea03040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T00:18:02.864 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.565+0000 7fd0b2a98040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:18:02.865 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.630+0000 7fd0b2a98040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:18:02.865 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.790+0000 7fd0b2a98040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:18:02.865 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.862+0000 7fd0b2a98040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:18:02.874 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.574+0000 7f78fea03040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T00:18:02.874 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.639+0000 7f78fea03040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T00:18:02.874 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.799+0000 7f78fea03040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T00:18:03.163 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:02.926+0000 7fd0b2a98040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:18:03.164 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.093+0000 7fd0b2a98040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:18:03.164 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.161+0000 7fd0b2a98040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:18:03.172 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.871+0000 7f78fea03040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T00:18:03.172 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:02.936+0000 7f78fea03040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T00:18:03.172 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.102+0000 7f78fea03040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T00:18:03.172 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.171+0000 7f78fea03040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T00:18:03.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.231+0000 7fd0b2a98040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:18:03.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.318+0000 7fd0b2a98040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:18:03.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.380+0000 7fd0b2a98040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:18:03.453 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.241+0000 7f78fea03040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T00:18:03.453 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.327+0000 7f78fea03040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T00:18:03.453 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.390+0000 7f78fea03040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T00:18:03.453 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:03 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:03.452+0000 7f78fea03040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:18:03.924 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:03.442+0000 7fd0b2a98040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T00:18:04.513 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:18:04.240Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:18:04.514 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:18:04.241Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:18:04.514 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:04.276+0000 7fd0b2a98040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:18:04.514 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:18:04] ENGINE Bus STARTING 2024-09-21T00:18:04.514 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: CherryPy Checker: 2024-09-21T00:18:04.514 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: The Application mounted at '' has an empty config. 2024-09-21T00:18:04.514 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: Active manager daemon y restarted 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: Activating manager daemon y 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: mgrmap e45: y(active, starting, since 0.00608082s), standbys: x 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: Standby manager daemon x restarted 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: Standby manager daemon x started 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:18:04.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:18:04.516 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[28324]: Manager daemon y is now available 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: Active manager daemon y restarted 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: Activating manager daemon y 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: mgrmap e45: y(active, starting, since 0.00608082s), standbys: x 2024-09-21T00:18:04.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: Standby manager daemon x restarted 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: Standby manager daemon x started 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:18:04.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:18:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:18:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-mon[78227]: Manager daemon y is now available 2024-09-21T00:18:04.545 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:04.285+0000 7f78fea03040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T00:18:04.545 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:18:04] ENGINE Bus STARTING 2024-09-21T00:18:04.545 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: CherryPy Checker: 2024-09-21T00:18:04.545 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: The Application mounted at '' has an empty config. 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:18:04] ENGINE Serving on http://:::9283 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x[60023]: [21/Sep/2024:00:18:04] ENGINE Bus STARTED 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: Active manager daemon y restarted 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: Activating manager daemon y 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: mgrmap e45: y(active, starting, since 0.00608082s), standbys: x 2024-09-21T00:18:04.546 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: Standby manager daemon x restarted 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: Standby manager daemon x started 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.? 172.21.15.31:0/1159973501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:18:04.547 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T00:18:04.548 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:04 smithi031 ceph-mon[63863]: Manager daemon y is now available 2024-09-21T00:18:04.775 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:18:04] ENGINE Serving on http://:::9283 2024-09-21T00:18:04.776 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:18:04] ENGINE Bus STARTED 2024-09-21T00:18:05.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:18:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:18:04.775Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2024-09-21T00:18:05.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:05.288+0000 7fd0517a1640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T00:18:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:18:05.532Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=3 2024-09-21T00:18:05.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:05.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:18:05.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:18:05.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[78227]: mgrmap e46: y(active, since 1.00885s), standbys: x 2024-09-21T00:18:05.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:05.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:18:05.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:18:05.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:05 smithi018 ceph-mon[28324]: mgrmap e46: y(active, since 1.00885s), standbys: x 2024-09-21T00:18:05.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:05.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T00:18:05.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T00:18:05.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:05 smithi031 ceph-mon[63863]: mgrmap e46: y(active, since 1.00885s), standbys: x 2024-09-21T00:18:06.674 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[28324]: [21/Sep/2024:00:18:06] ENGINE Bus STARTING 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[28324]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[28324]: Cluster is now healthy 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[78227]: [21/Sep/2024:00:18:06] ENGINE Bus STARTING 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[78227]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-mon[78227]: Cluster is now healthy 2024-09-21T00:18:06.675 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:06.295+0000 7fd00f055640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:00:18:06] ENGINE Error in HTTPServer.serve 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: Traceback (most recent call last): 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._connections.run(self.expiration_interval) 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._run(expiration_interval) 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: s = self.context.wrap_socket( 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:18:06.676 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: return self.sslsocket_class._create( 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self.do_handshake() 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: self._sslobj.do_handshake() 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:18:06.677 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:06.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:06 smithi031 ceph-mon[63863]: [21/Sep/2024:00:18:06] ENGINE Bus STARTING 2024-09-21T00:18:06.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:06 smithi031 ceph-mon[63863]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T00:18:06.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:06 smithi031 ceph-mon[63863]: Cluster is now healthy 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: [21/Sep/2024:00:18:06] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: [21/Sep/2024:00:18:06] ENGINE Error in HTTPServer.serve 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:18:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: self.do_handshake() 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: [21/Sep/2024:00:18:06] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[78227]: [21/Sep/2024:00:18:06] ENGINE Bus STARTED 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: [21/Sep/2024:00:18:06] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: [21/Sep/2024:00:18:06] ENGINE Error in HTTPServer.serve 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: Traceback (most recent call last): 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._run(expiration_interval) 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:18:07.677 INFO:journalctl@ceph.mon.c.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: self.do_handshake() 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: [21/Sep/2024:00:18:06] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:18:07.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:07 smithi018 ceph-mon[28324]: [21/Sep/2024:00:18:06] ENGINE Bus STARTED 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: [21/Sep/2024:00:18:06] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: [21/Sep/2024:00:18:06] ENGINE Error in HTTPServer.serve 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: Traceback (most recent call last): 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._connections.run(self.expiration_interval) 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._run(expiration_interval) 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T00:18:07.706 INFO:journalctl@ceph.mon.b.smithi031.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: s = self.context.wrap_socket( 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: return self.sslsocket_class._create( 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: self.do_handshake() 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: self._sslobj.do_handshake() 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: mgrmap e47: y(active, since 2s), standbys: x 2024-09-21T00:18:07.707 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: [21/Sep/2024:00:18:06] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T00:18:07.708 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:07 smithi031 ceph-mon[63863]: [21/Sep/2024:00:18:06] ENGINE Bus STARTED 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[78227]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[78227]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[28324]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[28324]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-21T00:18:09.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:09 smithi018 ceph-mon[28324]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:09 smithi031 ceph-mon[63863]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:09 smithi031 ceph-mon[63863]: mgrmap e48: y(active, since 4s), standbys: x 2024-09-21T00:18:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:09 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:10.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:10 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:12.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:12 smithi018 ceph-mon[78227]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:12.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:12 smithi018 ceph-mon[28324]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:12.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:18:11] "GET /metrics HTTP/1.1" 200 35205 "" "Prometheus/2.43.0" 2024-09-21T00:18:12.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:12 smithi031 ceph-mon[63863]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:18:14.169 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:18:14.169 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.171 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.172 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:13 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi031:/etc/ceph/ceph.conf 2024-09-21T00:18:14.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi031:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:14.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: Upgrade: It appears safe to stop mon.c 2024-09-21T00:18:15.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[28324]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: Upgrade: It appears safe to stop mon.c 2024-09-21T00:18:15.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:14 smithi018 ceph-mon[78227]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:18:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: Updating smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: Updating smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T00:18:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T00:18:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T00:18:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T00:18:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: Upgrade: It appears safe to stop mon.c 2024-09-21T00:18:15.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:14 smithi031 ceph-mon[63863]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: Upgrade: Updating mon.c 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[78227]: Deploying daemon mon.c on smithi018 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: Upgrade: Updating mon.c 2024-09-21T00:18:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:16.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:16.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:16.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:16.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 ceph-mon[28324]: Deploying daemon mon.c on smithi018 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: Upgrade: Updating mon.c 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:16.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:16 smithi031 ceph-mon[63863]: Deploying daemon mon.c on smithi018 2024-09-21T00:18:17.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:16 smithi018 systemd[1]: Stopping Ceph mon.c for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:18:17.413 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c[28320]: 2024-09-21T00:18:17.053+0000 7fa512507700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:18:17.413 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c[28320]: 2024-09-21T00:18:17.053+0000 7fa512507700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T00:18:17.413 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 podman[80399]: 2024-09-21 00:18:17.156769261 +0000 UTC m=+0.226011458 container died b12e4f83e07a37d9eb27c31dcfa7ceaa01a577db2422e8add4e17b10997d8657 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, release=754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T00:18:17.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 podman[80399]: 2024-09-21 00:18:17.50334407 +0000 UTC m=+0.572586257 container cleanup b12e4f83e07a37d9eb27c31dcfa7ceaa01a577db2422e8add4e17b10997d8657 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, version=8, maintainer=Guillaume Abrioux , ceph=True, name=centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T00:18:17.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 bash[80399]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c 2024-09-21T00:18:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 podman[80411]: 2024-09-21 00:18:17.779779637 +0000 UTC m=+0.621292041 container remove b12e4f83e07a37d9eb27c31dcfa7ceaa01a577db2422e8add4e17b10997d8657 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, ceph=True) 2024-09-21T00:18:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T00:18:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 systemd[1]: Stopped Ceph mon.c for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:18:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:17 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.c.service: Consumed 17.573s CPU time. 2024-09-21T00:18:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 systemd[1]: Starting Ceph mon.c for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:18:18.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 podman[80537]: 2024-09-21 00:18:18.258218147 +0000 UTC m=+0.016361101 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:18:18.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 podman[80537]: 2024-09-21 00:18:18.397329605 +0000 UTC m=+0.155472535 container create 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70) 2024-09-21T00:18:19.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 podman[80537]: 2024-09-21 00:18:18.685795885 +0000 UTC m=+0.443938831 container init 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:18:19.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 podman[80537]: 2024-09-21 00:18:18.689465582 +0000 UTC m=+0.447608517 container start 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:18:19.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T00:18:19.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable), process ceph-mon, pid 2 2024-09-21T00:18:19.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: pidfile_write: ignore empty --pid-file 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: load: jerasure load: lrc 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: RocksDB version: 7.9.2 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Git sha 0 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Compile date 2024-09-19 18:20:48 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: DB SUMMARY 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: DB Session ID: C2UAGLUXEACD0LMTBO09 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: CURRENT file: CURRENT 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3004 Bytes 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000057.sst 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000055.log size: 2320946 ; 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.error_if_exists: 0 2024-09-21T00:18:19.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.create_if_missing: 0 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.paranoid_checks: 1 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.env: 0x5555dca9d940 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.info_log: 0x5555df17f6e0 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T00:18:19.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.statistics: (nil) 2024-09-21T00:18:19.071 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.use_fsync: 0 2024-09-21T00:18:19.071 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_log_file_size: 0 2024-09-21T00:18:19.071 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T00:18:19.071 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_fallocate: 1 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.use_direct_reads: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.db_log_dir: 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.wal_dir: 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T00:18:19.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.write_buffer_manager: 0x5555df18edc0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T00:18:19.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.unordered_write: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.row_cache: None 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.wal_filter: None 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.two_write_queues: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.wal_compression: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.atomic_flush: 0 2024-09-21T00:18:19.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.log_readahead_size: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_background_jobs: 2 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_background_compactions: -1 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_subcompactions: 1 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T00:18:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_open_files: -1 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_background_flushes: -1 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Compression algorithms supported: 2024-09-21T00:18:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kZSTD supported: 0 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kXpressCompression supported: 0 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kBZip2Compression supported: 0 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kLZ4Compression supported: 1 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kZlibCompression supported: 1 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: kSnappyCompression supported: 1 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.merge_operator: 2024-09-21T00:18:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_filter: None 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5555df17f320) 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks: 1 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_top_level_index_and_filter: 1 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_type: 0 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_index_type: 0 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_shortening: 1 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T00:18:19.078 INFO:journalctl@ceph.mon.c.smithi018.stdout: checksum: 4 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: no_block_cache: 0 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache: 0x5555df1731f0 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_name: BinnedLRUCache 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_options: 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: capacity : 536870912 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: num_shard_bits : 4 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: strict_capacity_limit : 0 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: high_pri_pool_ratio: 0.000 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_compressed: (nil) 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: persistent_cache: (nil) 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size: 4096 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size_deviation: 10 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_restart_interval: 16 2024-09-21T00:18:19.079 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_block_restart_interval: 1 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: metadata_block_size: 4096 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: partition_filters: 0 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: use_delta_encoding: 1 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: filter_policy: bloomfilter 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: whole_key_filtering: 1 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: verify_compression: 0 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: read_amp_bytes_per_bit: 0 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: format_version: 5 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_index_compression: 1 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_align: 0 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: max_auto_readahead_size: 262144 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: prepopulate_block_cache: 0 2024-09-21T00:18:19.080 INFO:journalctl@ceph.mon.c.smithi018.stdout: initial_auto_readahead_size: 8192 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression: NoCompression 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.num_levels: 7 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T00:18:19.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T00:18:19.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T00:18:19.083 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T00:18:19.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.inplace_update_support: 0 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T00:18:19.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.bloom_locality: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.max_successive_merges: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.ttl: 2592000 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enable_blob_files: false 2024-09-21T00:18:19.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.min_blob_size: 0 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 57.sst 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 59, last_sequence is 30988, log_number is 55,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 55 2024-09-21T00:18:19.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7317ad47-9ba2-4360-9209-d4edbab880e5 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877898716927, "job": 1, "event": "recovery_started", "wal_files": [55]} 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #55 mode 2 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877898723285, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 60, "file_size": 1479659, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 30994, "largest_seqno": 31376, "table_properties": {"data_size": 1476761, "index_size": 1346, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4517, "raw_average_key_size": 24, "raw_value_size": 1472453, "raw_average_value_size": 7916, "num_data_blocks": 60, "num_entries": 186, "num_filter_entries": 186, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726877898, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7317ad47-9ba2-4360-9209-d4edbab880e5", "db_session_id": "C2UAGLUXEACD0LMTBO09", "orig_file_number": 60, "seqno_to_time_mapping": "N/A"}} 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726877898723403, "job": 1, "event": "recovery_finished"} 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:5047] Creating manifest 62 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000055.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5555df1a0e00 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: DB pointer 0x5555df29c000 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T00:18:19.088 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** DB Stats ** 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: L0 1/0 1.41 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 473.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: L6 1/0 10.61 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T00:18:19.089 INFO:journalctl@ceph.mon.c.smithi018.stdout: Sum 2/0 12.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 473.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 473.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 473.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T00:18:19.090 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative compaction: 0.00 GB write, 159.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval compaction: 0.00 GB write, 159.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: Block cache BinnedLRUCache@0x5555df1731f0#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: starting mon.c rank 1 at public addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] at bind addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???) e3 preinit fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).mds e1 new map 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).mds e1 print_map 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: e1 2024-09-21T00:18:19.091 INFO:journalctl@ceph.mon.c.smithi018.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout: legacy client fscid: -1 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout: No filesystems configured 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).osd e114 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 ceph-mon[80555]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 bash[80537]: 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 2024-09-21T00:18:19.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:18 smithi018 systemd[1]: Started Ceph mon.c for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:18:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:20 smithi018 ceph-mon[78227]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:18:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:20 smithi018 ceph-mon[80555]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:18:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:20 smithi031 ceph-mon[63863]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-21T00:18:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:18:21] "GET /metrics HTTP/1.1" 200 37943 "" "Prometheus/2.43.0" 2024-09-21T00:18:24.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:23 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:18:23.733+0000 7fd0517a1640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T00:18:26.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mon.c calling monitor election 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mon.b calling monitor election 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: monmap epoch 4 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: min_mon_release 19 (squid) 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: election_strategy: 1 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: fsmap 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mon.c calling monitor election 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mon.a calling monitor election 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: monmap epoch 4 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: min_mon_release 19 (squid) 2024-09-21T00:18:26.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: election_strategy: 1 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: fsmap 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:26.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:26 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:26.538 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:18:26.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mon.c calling monitor election 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mon.b calling monitor election 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: monmap epoch 4 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: min_mon_release 19 (squid) 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: election_strategy: 1 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: fsmap 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mon.c calling monitor election 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mon.a calling monitor election 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: monmap epoch 4 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: min_mon_release 19 (squid) 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: election_strategy: 1 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: fsmap 2024-09-21T00:18:26.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mon.c calling monitor election 2024-09-21T00:18:26.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mon.b calling monitor election 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: monmap epoch 4 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: min_mon_release 19 (squid) 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: election_strategy: 1 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: fsmap 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mon.c calling monitor election 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mon.a calling monitor election 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: monmap epoch 4 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: fsid c1f9e122-77aa-11ef-baf5-efdc52797490 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: last_changed 2024-09-21T00:18:20.173081+0000 2024-09-21T00:18:26.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: created 2024-09-20T23:48:25.756842+0000 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: min_mon_release 19 (squid) 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: election_strategy: 1 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.a 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 1: [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon.c 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: 2: [v2:172.21.15.31:3300/0,v1:172.21.15.31:6789/0] mon.b 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: fsmap 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: mgrmap e48: y(active, since 20s), standbys: x 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:26.545 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:26 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:27.242 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (15m) 17s ago 23m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (2m) 18s ago 23m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 17s ago 22m 60.3M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (2m) 18s ago 28m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (12m) 17s ago 29m 473M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (29s) 17s ago 29m 29.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (79s) 18s ago 28m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 starting - - - 2048M 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (15m) 17s ago 24m 16.3M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (15m) 18s ago 24m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:18:27.243 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (27m) 17s ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (27m) 17s ago 27m 55.9M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (27m) 17s ago 27m 52.9M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (26m) 17s ago 26m 53.9M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (26m) 18s ago 26m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (25m) 18s ago 25m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (25m) 18s ago 25m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (25m) 18s ago 25m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (2m) 18s ago 24m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (22m) 17s ago 22m 79.4M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:18:27.244 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (22m) 18s ago 22m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:18:27.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:27 smithi031 ceph-mon[63863]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:27.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:27 smithi018 ceph-mon[80555]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:27.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:27 smithi018 ceph-mon[78227]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:27.674 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:18:27.675 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:18:27.676 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T00:18:27.676 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 5 2024-09-21T00:18:27.676 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:18:27.676 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:18:28.032 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:18:28.032 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:18:28.032 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:18:28.032 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi018", 2024-09-21T00:18:28.032 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:18:28.033 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "1/2 daemons upgraded", 2024-09-21T00:18:28.033 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T00:18:28.033 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:18:28.033 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:18:28.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[78227]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[78227]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[78227]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3238111118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:28.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[80555]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[80555]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[80555]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:28 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3238111118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:28 smithi031 ceph-mon[63863]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:28 smithi031 ceph-mon[63863]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:28 smithi031 ceph-mon[63863]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:28 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3238111118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:29 smithi018 ceph-mon[78227]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:29 smithi018 ceph-mon[78227]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:29 smithi018 ceph-mon[80555]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:29 smithi018 ceph-mon[80555]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:29 smithi031 ceph-mon[63863]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:29 smithi031 ceph-mon[63863]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:30.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:30 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:30.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:30.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:30.674 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:30.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:30.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:30.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:30.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:30.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:31 smithi031 ceph-mon[63863]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:31 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:31 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.543 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[78227]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:31.543 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.544 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[80555]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:31.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:31.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:32.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:18:31] "GET /metrics HTTP/1.1" 200 37943 "" "Prometheus/2.43.0" 2024-09-21T00:18:33.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:33 smithi031 ceph-mon[63863]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:33.511 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:33 smithi018 ceph-mon[80555]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:33.512 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:33 smithi018 ceph-mon[78227]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: Detected new or changed devices on smithi018 2024-09-21T00:18:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: Reconfiguring mon.a (monmap changed)... 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:34.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: Reconfiguring daemon mon.a on smithi018 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: Detected new or changed devices on smithi018 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:34.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: Reconfiguring mon.a (monmap changed)... 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:34.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:34.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: Reconfiguring daemon mon.a on smithi018 2024-09-21T00:18:34.929 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: Detected new or changed devices on smithi018 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: Reconfiguring mon.a (monmap changed)... 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: Reconfiguring daemon mon.a on smithi018 2024-09-21T00:18:35.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:36.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: Reconfiguring daemon mgr.y on smithi018 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[80555]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.652 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T00:18:36.653 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:36.653 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:36.653 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:36.653 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: Reconfiguring daemon mgr.y on smithi018 2024-09-21T00:18:36.653 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:36 smithi018 ceph-mon[78227]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:36.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:36.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T00:18:36.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:36.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:36.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:36.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: Reconfiguring daemon mgr.y on smithi018 2024-09-21T00:18:36.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:36 smithi031 ceph-mon[63863]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: Reconfiguring mon.c (monmap changed)... 2024-09-21T00:18:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:38.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:37 smithi031 ceph-mon[63863]: Reconfiguring daemon mon.c on smithi018 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: Reconfiguring mon.c (monmap changed)... 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:38.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[80555]: Reconfiguring daemon mon.c on smithi018 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: Reconfiguring mon.c (monmap changed)... 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:38.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:37 smithi018 ceph-mon[78227]: Reconfiguring daemon mon.c on smithi018 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.0 on smithi018 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:39.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T00:18:39.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:18:39.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:39.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:39 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.0 on smithi018 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:39 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.0 on smithi018 2024-09-21T00:18:40.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:40.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:18:40.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:40 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:18:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.1 on smithi018 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:18:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.1 on smithi018 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:18:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.1 on smithi018 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:18:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:42.158 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:18:41] "GET /metrics HTTP/1.1" 200 38025 "" "Prometheus/2.43.0" 2024-09-21T00:18:42.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:42 smithi018 ceph-mon[78227]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T00:18:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:42 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.2 on smithi018 2024-09-21T00:18:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:42 smithi018 ceph-mon[80555]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T00:18:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:42 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.2 on smithi018 2024-09-21T00:18:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:42 smithi031 ceph-mon[63863]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T00:18:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:42 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.2 on smithi018 2024-09-21T00:18:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[80555]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:18:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:43.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[78227]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:43.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:18:43.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:43 smithi031 ceph-mon[63863]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:18:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.3 on smithi018 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.3 on smithi018 2024-09-21T00:18:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.3 on smithi018 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:44.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[80555]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[80555]: Reconfiguring rgw.foo.smithi018.kionxd (monmap changed)... 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[80555]: Reconfiguring daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[78227]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[78227]: Reconfiguring rgw.foo.smithi018.kionxd (monmap changed)... 2024-09-21T00:18:45.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:45 smithi018 ceph-mon[78227]: Reconfiguring daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:18:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:45 smithi031 ceph-mon[63863]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:18:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:45 smithi031 ceph-mon[63863]: Reconfiguring rgw.foo.smithi018.kionxd (monmap changed)... 2024-09-21T00:18:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:45 smithi031 ceph-mon[63863]: Reconfiguring daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: Reconfiguring mon.b (monmap changed)... 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:46.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: Reconfiguring daemon mon.b on smithi031 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[78227]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: Reconfiguring mon.b (monmap changed)... 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: Reconfiguring daemon mon.b on smithi031 2024-09-21T00:18:46.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:46 smithi018 ceph-mon[80555]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: Reconfiguring mon.b (monmap changed)... 2024-09-21T00:18:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T00:18:46.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T00:18:46.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:46.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: Reconfiguring daemon mon.b on smithi031 2024-09-21T00:18:46.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:46 smithi031 ceph-mon[63863]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:47 smithi031 ceph-mon[63863]: Reconfiguring daemon mgr.x on smithi031 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[78227]: Reconfiguring daemon mgr.x on smithi031 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T00:18:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T00:18:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T00:18:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:48.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:47 smithi018 ceph-mon[80555]: Reconfiguring daemon mgr.x on smithi031 2024-09-21T00:18:49.036 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:48 smithi031 ceph-mon[63863]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:49.036 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.036 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.036 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:18:49.036 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:49.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[80555]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:49.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[78227]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:18:49.328 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.4 on smithi031 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:18:50.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:50.343 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T00:18:50.343 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.4 on smithi031 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.4 on smithi031 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:50.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:18:50.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:51.166 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:50 smithi031 ceph-mon[63863]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T00:18:51.166 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:50 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.5 on smithi031 2024-09-21T00:18:51.166 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:50 smithi031 ceph-mon[63863]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[78227]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.5 on smithi031 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[78227]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[80555]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.5 on smithi031 2024-09-21T00:18:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:50 smithi018 ceph-mon[80555]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:18:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:18:51] "GET /metrics HTTP/1.1" 200 38022 "" "Prometheus/2.43.0" 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.6 on smithi031 2024-09-21T00:18:52.606 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:52 smithi031 ceph-mon[63863]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.6 on smithi031 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[80555]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.6 on smithi031 2024-09-21T00:18:52.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:52 smithi018 ceph-mon[78227]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:53.937 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:53 smithi031 ceph-mon[63863]: Reconfiguring daemon osd.7 on smithi031 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[80555]: Reconfiguring daemon osd.7 on smithi031 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:54.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:54.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T00:18:54.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:18:54.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:54.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:53 smithi018 ceph-mon[78227]: Reconfiguring daemon osd.7 on smithi031 2024-09-21T00:18:55.343 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:55.343 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.343 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.343 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: Reconfiguring rgw.foo.smithi031.jzvksl (monmap changed)... 2024-09-21T00:18:55.343 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:55.344 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:55.344 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:55 smithi031 ceph-mon[63863]: Reconfiguring daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: Reconfiguring rgw.foo.smithi031.jzvksl (monmap changed)... 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[80555]: Reconfiguring daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:18:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:55.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: Reconfiguring rgw.foo.smithi031.jzvksl (monmap changed)... 2024-09-21T00:18:55.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:18:55.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:55.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:55 smithi018 ceph-mon[78227]: Reconfiguring daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:18:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mds 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nfs 2024-09-21T00:18:56.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mds 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:18:56.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nfs 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all grafana 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all loki 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all promtail 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Finalizing container_image settings 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:18:56.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all grafana 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all loki 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all promtail 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Finalizing container_image settings 2024-09-21T00:18:56.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: Upgrade: Complete! 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:18:56.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:18:56.688 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:18:56.689 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: Upgrade: Complete! 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.691 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.691 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.691 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:18:56.691 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:56 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mds 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nfs 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all node-exporter 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.754 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all prometheus 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all alertmanager 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all grafana 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all loki 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.755 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all promtail 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:18:56.756 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.757 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.758 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:18:56.759 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:56 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:58.961 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:58 smithi018 ceph-mon[78227]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:18:58.962 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:58 smithi018 ceph-mon[80555]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:18:58.986 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:18:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:58 smithi031 ceph-mon[63863]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[78227]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[80555]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:18:59.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:18:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:00.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:59 smithi031 ceph-mon[63863]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:00.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:59 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:00.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:18:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:00.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:00 smithi018 ceph-mon[78227]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:19:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:00 smithi018 ceph-mon[80555]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (16m) 30s ago 24m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (2m) 52s ago 23m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 30s ago 23m 61.3M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (3m) 52s ago 28m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (12m) 30s ago 30m 491M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (63s) 30s ago 30m 40.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (113s) 52s ago 29m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:19:00.980 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (42s) 30s ago 29m 21.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (16m) 30s ago 24m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (15m) 52s ago 24m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (28m) 30s ago 28m 54.8M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (28m) 30s ago 28m 56.1M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (27m) 30s ago 27m 53.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (27m) 30s ago 27m 53.7M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (26m) 52s ago 26m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (26m) 52s ago 26m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (26m) 52s ago 26m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (25m) 52s ago 25m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:19:00.981 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (3m) 52s ago 25m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:19:00.982 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (23m) 30s ago 23m 79.7M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:19:00.982 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (23m) 52s ago 23m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:19:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:00 smithi031 ceph-mon[63863]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 882 B/s rd, 0 op/s 2024-09-21T00:19:01.632 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-09-21T00:19:01.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:01 smithi018 ceph-mon[78227]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:01.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:01 smithi018 ceph-mon[80555]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:02.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:01 smithi031 ceph-mon[63863]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:01] "GET /metrics HTTP/1.1" 200 38022 "" "Prometheus/2.43.0" 2024-09-21T00:19:02.988 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:02 smithi018 ceph-mon[78227]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:19:02.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:02 smithi018 ceph-mon[80555]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:19:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:02 smithi031 ceph-mon[63863]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:19:03.678 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:19:03.963 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:03 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/545362784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:03.963 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:03 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/545362784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:04.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:03 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/545362784' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:04.214 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-09-21T00:19:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:04 smithi018 ceph-mon[78227]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:19:04.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:04.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:04 smithi018 ceph-mon[80555]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:19:04.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:04 smithi031 ceph-mon[63863]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-21T00:19:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:06.487 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-21T00:19:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:06 smithi031 ceph-mon[63863]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:06 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2718388133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:07.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:06 smithi018 ceph-mon[78227]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:07.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:06 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2718388133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:07.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:06 smithi018 ceph-mon[80555]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:07.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:06 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2718388133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:07.111 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-09-21T00:19:08.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:08 smithi018 ceph-mon[78227]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:08 smithi018 ceph-mon[80555]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:08 smithi031 ceph-mon[63863]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:10.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:09 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:10.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:09 smithi018 ceph-mon[78227]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:10.151 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:09 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:10.151 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:09 smithi018 ceph-mon[80555]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:09 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:10.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:09 smithi031 ceph-mon[63863]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:10.479 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:19:11.114 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:10 smithi018 ceph-mon[78227]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:11.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:10 smithi018 ceph-mon[80555]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:11.141 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:19:11.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:10 smithi031 ceph-mon[63863]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:12.095 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:11] "GET /metrics HTTP/1.1" 200 38023 "" "Prometheus/2.43.0" 2024-09-21T00:19:12.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:12 smithi018 ceph-mon[78227]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:12.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:12 smithi018 ceph-mon[80555]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:19:13.175 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:19:13.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:12 smithi031 ceph-mon[63863]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:13.801 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:19:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:13 smithi031 ceph-mon[63863]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:14.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:13 smithi018 ceph-mon[78227]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:14.108 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:13 smithi018 ceph-mon[80555]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:15.116 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:14 smithi018 ceph-mon[80555]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:15.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:14 smithi018 ceph-mon[78227]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:15.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:14 smithi031 ceph-mon[63863]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:15.908 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:19:15.908 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:19:15.908 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:19:16.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-09-21T00:19:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:16 smithi018 ceph-mon[78227]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:16 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3528940248' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:19:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:16 smithi018 ceph-mon[80555]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:16 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3528940248' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:19:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:16 smithi031 ceph-mon[63863]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:16 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3528940248' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:19:19.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:18 smithi018 ceph-mon[78227]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:19.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:18 smithi018 ceph-mon[80555]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:18 smithi031 ceph-mon[63863]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:19.935 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[78227]: from='client.44208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[80555]: from='client.44208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:20.064 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:19 smithi031 ceph-mon[63863]: from='client.44208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:19 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:20.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:19:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:21.104 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.008 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:21] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-09-21T00:19:22.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:22.009 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:19:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:23.057 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mgr 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[78227]: Upgrade: osd.0 is safe to restart 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:19:23.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mgr 2024-09-21T00:19:23.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:19:23.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:19:23.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:23.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:22 smithi018 ceph-mon[80555]: Upgrade: osd.0 is safe to restart 2024-09-21T00:19:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:19:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:19:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:19:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:19:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:19:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T00:19:23.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:22 smithi031 ceph-mon[63863]: Upgrade: osd.0 is safe to restart 2024-09-21T00:19:23.564 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:19:24.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:23 smithi018 ceph-mon[78227]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:24.064 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:23 smithi018 ceph-mon[80555]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:24.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:23 smithi031 ceph-mon[63863]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:24.257 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:19:24.257 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (16m) 54s ago 24m 23.5M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:19:24.257 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (3m) 75s ago 24m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:19:24.257 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 54s ago 23m 61.3M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (3m) 75s ago 29m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (13m) 54s ago 30m 491M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (86s) 54s ago 30m 40.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (2m) 75s ago 29m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (65s) 54s ago 29m 21.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (16m) 54s ago 25m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (16m) 75s ago 25m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (28m) 54s ago 28m 54.8M 2993M 17.2.0 e1d6a67b021e 1ac6545048b5 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (28m) 54s ago 28m 56.1M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (28m) 54s ago 28m 53.2M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (27m) 54s ago 27m 53.7M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (27m) 75s ago 27m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:19:24.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (26m) 75s ago 26m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:19:24.259 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (26m) 75s ago 26m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:19:24.259 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (26m) 75s ago 26m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:19:24.259 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (3m) 75s ago 25m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:19:24.259 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (23m) 54s ago 23m 79.7M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:19:24.259 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (23m) 75s ago 23m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:19:24.694 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:19:24.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:19:24.694 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:19:24.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 5 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:19:24.695 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:19:25.054 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/8 daemons upgraded", 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:19:25.055 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:19:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:19:25.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:25.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:24 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3397246781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:25.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3397246781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='client.44214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:25.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:25.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T00:19:25.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:25.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:24 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3397246781' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[78227]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[78227]: Upgrade: Updating osd.0 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[78227]: Deploying daemon osd.0 on smithi018 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[78227]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[80555]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[80555]: Upgrade: Updating osd.0 2024-09-21T00:19:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[80555]: Deploying daemon osd.0 on smithi018 2024-09-21T00:19:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:25 smithi018 ceph-mon[80555]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:25 smithi031 ceph-mon[63863]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-09-21T00:19:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:25 smithi031 ceph-mon[63863]: Upgrade: Updating osd.0 2024-09-21T00:19:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:25 smithi031 ceph-mon[63863]: Deploying daemon osd.0 on smithi018 2024-09-21T00:19:26.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:25 smithi031 ceph-mon[63863]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:27.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:27 smithi018 systemd[1]: Stopping Ceph osd.0 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:19:27.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[31152]: 2024-09-21T00:19:27.634+0000 7ff949a7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:19:27.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[31152]: 2024-09-21T00:19:27.634+0000 7ff949a7d700 -1 osd.0 114 *** Got signal Terminated *** 2024-09-21T00:19:27.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[31152]: 2024-09-21T00:19:27.634+0000 7ff949a7d700 -1 osd.0 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:19:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:27 smithi031 ceph-mon[63863]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:28.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:27 smithi031 ceph-mon[63863]: osd.0 marked itself down and dead 2024-09-21T00:19:28.300 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-mon[78227]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:28.300 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-mon[78227]: osd.0 marked itself down and dead 2024-09-21T00:19:28.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-mon[80555]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:28.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:27 smithi018 ceph-mon[80555]: osd.0 marked itself down and dead 2024-09-21T00:19:28.577 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:28 smithi018 podman[87452]: 2024-09-21 00:19:28.299359048 +0000 UTC m=+0.760330198 container died 1ac6545048b514ba37cae87a42f82babe46f7a270c90bc1869b29ea7105aad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, io.openshift.tags=base centos centos-stream, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, architecture=x86_64, distribution-scope=public) 2024-09-21T00:19:28.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:28 smithi018 podman[87452]: 2024-09-21 00:19:28.650613148 +0000 UTC m=+1.111584285 container cleanup 1ac6545048b514ba37cae87a42f82babe46f7a270c90bc1869b29ea7105aad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, vcs-type=git, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, name=centos-stream) 2024-09-21T00:19:28.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:28 smithi018 bash[87452]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0 2024-09-21T00:19:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:28 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:19:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:28 smithi031 ceph-mon[63863]: osdmap e115: 8 total, 7 up, 8 in 2024-09-21T00:19:29.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:28 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:19:29.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:28 smithi018 ceph-mon[78227]: osdmap e115: 8 total, 7 up, 8 in 2024-09-21T00:19:29.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:28 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:19:29.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:28 smithi018 ceph-mon[80555]: osdmap e115: 8 total, 7 up, 8 in 2024-09-21T00:19:29.291 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:28 smithi018 podman[87467]: 2024-09-21 00:19:28.926473804 +0000 UTC m=+0.625934613 container remove 1ac6545048b514ba37cae87a42f82babe46f7a270c90bc1869b29ea7105aad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-type=git) 2024-09-21T00:19:29.564 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.193505488 +0000 UTC m=+0.017248598 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:29.564 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.327922251 +0000 UTC m=+0.151665347 container create 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T00:19:29.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.563828137 +0000 UTC m=+0.387571245 container init 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T00:19:29.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.567320866 +0000 UTC m=+0.391063968 container start 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:19:29.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.638792619 +0000 UTC m=+0.462535725 container attach 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T00:19:29.925 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:29 smithi018 podman[87557]: 2024-09-21 00:19:29.76650329 +0000 UTC m=+0.590246394 container died 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T00:19:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:29 smithi031 ceph-mon[63863]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:19:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:29 smithi031 ceph-mon[63863]: osdmap e116: 8 total, 7 up, 8 in 2024-09-21T00:19:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:29 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:30.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[80555]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:19:30.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[80555]: osdmap e116: 8 total, 7 up, 8 in 2024-09-21T00:19:30.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:30.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[78227]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:19:30.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[78227]: osdmap e116: 8 total, 7 up, 8 in 2024-09-21T00:19:30.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:29 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:30.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 podman[87579]: 2024-09-21 00:19:30.283727743 +0000 UTC m=+0.509520997 container remove 9b6d96c83796ae972b3dc9093a32eed6e9e759d8a74178023539df46330ba91a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, ceph=True) 2024-09-21T00:19:30.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T00:19:30.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 systemd[1]: Stopped Ceph osd.0 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:19:30.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.0.service: Consumed 15.985s CPU time. 2024-09-21T00:19:30.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 systemd[1]: Starting Ceph osd.0 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:19:31.019 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 podman[87679]: 2024-09-21 00:19:30.644680257 +0000 UTC m=+0.016605662 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:31.019 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:30 smithi018 podman[87679]: 2024-09-21 00:19:30.759979901 +0000 UTC m=+0.131905302 container create e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T00:19:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:31 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:19:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:31 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:19:31.425 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 podman[87679]: 2024-09-21 00:19:31.022963077 +0000 UTC m=+0.394888472 container init e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T00:19:31.425 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 podman[87679]: 2024-09-21 00:19:31.026804617 +0000 UTC m=+0.398730013 container start e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T00:19:31.425 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 podman[87679]: 2024-09-21 00:19:31.106289147 +0000 UTC m=+0.478214555 container attach e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731) 2024-09-21T00:19:31.426 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:19:31.426 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 bash[87679]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:19:31.426 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:19:31.426 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:31 smithi018 bash[87679]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:19:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:31 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:19:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-mon[78227]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-mon[80555]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:31] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-09-21T00:19:32.409 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:32 smithi031 ceph-mon[63863]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:32.859 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T00:19:32.859 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T00:19:32.859 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T00:19:32.859 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate[87693]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T00:19:33.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 bash[87679]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T00:19:33.176 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:32 smithi018 podman[87679]: 2024-09-21 00:19:32.887776175 +0000 UTC m=+2.259701572 container died e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T00:19:33.784 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 podman[87834]: 2024-09-21 00:19:33.468351236 +0000 UTC m=+0.571129137 container remove e97fe14372104a9fcd9d6e2fa44b40073bb20a0dc7e9b4bd5aaec2d95691335a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T00:19:33.784 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 podman[87896]: 2024-09-21 00:19:33.677721799 +0000 UTC m=+0.113557869 container create 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T00:19:33.784 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 podman[87896]: 2024-09-21 00:19:33.580611056 +0000 UTC m=+0.016447087 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:19:34.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:34 smithi018 ceph-mon[78227]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:34.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:34 smithi018 ceph-mon[80555]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:34.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 podman[87896]: 2024-09-21 00:19:33.888264693 +0000 UTC m=+0.324100722 container init 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T00:19:34.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 podman[87896]: 2024-09-21 00:19:33.891536526 +0000 UTC m=+0.327372555 container start 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T00:19:34.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 ceph-osd[87915]: -- 172.21.15.18:0/3200055838 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e623511a40 con 0x55e6236c8400 2024-09-21T00:19:34.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 bash[87896]: 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 2024-09-21T00:19:34.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:33 smithi018 systemd[1]: Started Ceph osd.0 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:19:34.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:34 smithi031 ceph-mon[63863]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:35.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:35.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:35.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:35 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:19:35.564+0000 7feb87988640 -1 Falling back to public interface 2024-09-21T00:19:36.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:36 smithi018 ceph-mon[78227]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:36.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:36 smithi018 ceph-mon[80555]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:36.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:36 smithi031 ceph-mon[63863]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:38.367 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:38 smithi018 ceph-mon[78227]: pgmap v58: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 490 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:38.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:38 smithi018 ceph-mon[80555]: pgmap v58: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 490 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:38.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:38 smithi031 ceph-mon[63863]: pgmap v58: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 490 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:39.425 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:19:39.085+0000 7feb87988640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-21T00:19:39.425 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:19:39.119+0000 7feb87988640 -1 osd.0 114 log_to_monitors true 2024-09-21T00:19:40.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:40.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T00:19:40.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.177 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:19:40 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:19:40.071+0000 7feb7f733640 -1 osd.0 114 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[78227]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T00:19:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[78227]: osdmap e117: 8 total, 7 up, 8 in 2024-09-21T00:19:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[78227]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:19:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[80555]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T00:19:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[80555]: osdmap e117: 8 total, 7 up, 8 in 2024-09-21T00:19:41.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-mon[80555]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:19:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:41 smithi031 ceph-mon[63863]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T00:19:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:41 smithi031 ceph-mon[63863]: osdmap e117: 8 total, 7 up, 8 in 2024-09-21T00:19:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:41 smithi031 ceph-mon[63863]: from='osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:19:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:41] "GET /metrics HTTP/1.1" 200 38150 "" "Prometheus/2.43.0" 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: pgmap v61: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699] boot 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[78227]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: pgmap v61: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:19:42.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699] boot 2024-09-21T00:19:42.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T00:19:42.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:19:42.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-mon[80555]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T00:19:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: pgmap v61: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:19:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: osd.0 [v2:172.21.15.18:6802/2153660699,v1:172.21.15.18:6803/2153660699] boot 2024-09-21T00:19:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T00:19:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T00:19:42.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:42 smithi031 ceph-mon[63863]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T00:19:42.924 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:42 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:19:42.804+0000 7fd0517a1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T00:19:43.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: pgmap v65: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: pgmap v66: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:43 smithi031 ceph-mon[63863]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: pgmap v65: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: pgmap v66: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[78227]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: pgmap v65: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: pgmap v66: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:43.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:43 smithi018 ceph-mon[80555]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T00:19:46.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:45 smithi031 ceph-mon[63863]: pgmap v67: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:46.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:45 smithi018 ceph-mon[78227]: pgmap v67: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:46.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:45 smithi018 ceph-mon[80555]: pgmap v67: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-21T00:19:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:46 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded) 2024-09-21T00:19:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:46 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded) 2024-09-21T00:19:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:46 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 18 pgs degraded) 2024-09-21T00:19:48.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:47 smithi018 ceph-mon[78227]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-09-21T00:19:48.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:47 smithi018 ceph-mon[80555]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-09-21T00:19:48.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:47 smithi031 ceph-mon[63863]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[78227]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[80555]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:49 smithi031 ceph-mon[63863]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 772 B/s rd, 0 op/s 2024-09-21T00:19:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:49 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:19:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:19:52.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:51 smithi018 ceph-mon[78227]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:52.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:51 smithi018 ceph-mon[80555]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:19:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:19:51] "GET /metrics HTTP/1.1" 200 38162 "" "Prometheus/2.43.0" 2024-09-21T00:19:52.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:51 smithi031 ceph-mon[63863]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:54.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:53 smithi018 ceph-mon[78227]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:53 smithi018 ceph-mon[80555]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:53 smithi031 ceph-mon[63863]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:19:55.437 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:19:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:55 smithi018 ceph-mon[80555]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:55 smithi018 ceph-mon[78227]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:19:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:55 smithi031 ceph-mon[63863]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (17m) 17s ago 25m 23.7M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (3m) 107s ago 24m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 17s ago 24m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (4m) 107s ago 29m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (13m) 17s ago 31m 498M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:19:56.121 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (118s) 17s ago 31m 45.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (2m) 107s ago 29m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (97s) 17s ago 30m 37.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (17m) 17s ago 25m 16.7M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (16m) 107s ago 25m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (22s) 17s ago 29m 11.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (29m) 17s ago 29m 55.7M 2993M 17.2.0 e1d6a67b021e ba7ce69c0824 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (28m) 17s ago 28m 54.0M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (28m) 17s ago 28m 54.5M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (27m) 107s ago 27m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (27m) 107s ago 27m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (27m) 107s ago 27m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:19:56.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (26m) 107s ago 26m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:19:56.123 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (4m) 107s ago 25m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:19:56.123 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (24m) 17s ago 24m 79.9M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:19:56.123 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (24m) 107s ago 24m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:19:56.536 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:19:56.536 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:19:56.536 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:19:56.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 1 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 6 2024-09-21T00:19:56.537 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:19:56.538 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:19:56.674 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[78227]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[78227]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[78227]: from='client.54199 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[80555]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[80555]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:56 smithi018 ceph-mon[80555]: from='client.54199 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:56 smithi031 ceph-mon[63863]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:56 smithi031 ceph-mon[63863]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:56 smithi031 ceph-mon[63863]: from='client.54199 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "1/8 daemons upgraded", 2024-09-21T00:19:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:19:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:19:56.887 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3202003314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[78227]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[78227]: from='client.54211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3202003314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[80555]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:57 smithi018 ceph-mon[80555]: from='client.54211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:57 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3202003314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:19:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:57 smithi031 ceph-mon[63863]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:19:57.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:57 smithi031 ceph-mon[63863]: from='client.54211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:19:58.510 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.510 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.510 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[78227]: Upgrade: osd.1 is safe to restart 2024-09-21T00:19:58.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.511 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:19:58 smithi018 ceph-mon[80555]: Upgrade: osd.1 is safe to restart 2024-09-21T00:19:58.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:58 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:58 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T00:19:58.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:19:58 smithi031 ceph-mon[63863]: Upgrade: osd.1 is safe to restart 2024-09-21T00:20:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: Upgrade: Updating osd.1 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: Deploying daemon osd.1 on smithi018 2024-09-21T00:20:00.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:00 smithi031 ceph-mon[63863]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: Upgrade: Updating osd.1 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: Deploying daemon osd.1 on smithi018 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[78227]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:20:00.565 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: Upgrade: Updating osd.1 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: Deploying daemon osd.1 on smithi018 2024-09-21T00:20:00.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:00 smithi018 ceph-mon[80555]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:20:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:01] "GET /metrics HTTP/1.1" 200 38162 "" "Prometheus/2.43.0" 2024-09-21T00:20:02.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:02 smithi018 ceph-mon[78227]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:02.462 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:02 smithi018 ceph-mon[80555]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:02.462 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:02 smithi018 systemd[1]: Stopping Ceph osd.1 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:20:02.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:02 smithi031 ceph-mon[63863]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:02.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[34112]: 2024-09-21T00:20:02.460+0000 7f1519cbf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:20:02.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[34112]: 2024-09-21T00:20:02.460+0000 7f1519cbf700 -1 osd.1 119 *** Got signal Terminated *** 2024-09-21T00:20:02.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[34112]: 2024-09-21T00:20:02.460+0000 7f1519cbf700 -1 osd.1 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:20:03.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:03 smithi031 ceph-mon[63863]: osd.1 marked itself down and dead 2024-09-21T00:20:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:03 smithi018 ceph-mon[78227]: osd.1 marked itself down and dead 2024-09-21T00:20:03.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:03 smithi018 ceph-mon[80555]: osd.1 marked itself down and dead 2024-09-21T00:20:03.961 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:03 smithi018 podman[92301]: 2024-09-21 00:20:03.708124903 +0000 UTC m=+1.366086712 container died ba7ce69c0824e991abc7791ad95f4e29eb3e4bd82ebc7a83369f5717ca707e49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, RELEASE=HEAD, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-09-21T00:20:04.225 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92301]: 2024-09-21 00:20:04.051020946 +0000 UTC m=+1.708982739 container cleanup ba7ce69c0824e991abc7791ad95f4e29eb3e4bd82ebc7a83369f5717ca707e49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-09-21T00:20:04.225 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 bash[92301]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1 2024-09-21T00:20:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:04 smithi031 ceph-mon[63863]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:04 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:04.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:04 smithi031 ceph-mon[63863]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[78227]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[78227]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[80555]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:04 smithi018 ceph-mon[80555]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T00:20:04.675 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92317]: 2024-09-21 00:20:04.260707616 +0000 UTC m=+0.547698131 container remove ba7ce69c0824e991abc7791ad95f4e29eb3e4bd82ebc7a83369f5717ca707e49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, release=754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T00:20:05.006 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92410]: 2024-09-21 00:20:04.583775503 +0000 UTC m=+0.016705848 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:05.007 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92410]: 2024-09-21 00:20:04.703617643 +0000 UTC m=+0.136547967 container create 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T00:20:05.007 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92410]: 2024-09-21 00:20:04.922737223 +0000 UTC m=+0.355667566 container init 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T00:20:05.007 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:04 smithi018 podman[92410]: 2024-09-21 00:20:04.926262106 +0000 UTC m=+0.359192440 container start 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:20:05.007 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 podman[92410]: 2024-09-21 00:20:05.005993429 +0000 UTC m=+0.438923759 container attach 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T00:20:05.411 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T00:20:05.411 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]: dispatch 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:05.412 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 podman[92410]: 2024-09-21 00:20:05.124807355 +0000 UTC m=+0.557737690 container died 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]: dispatch 2024-09-21T00:20:05.413 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]: dispatch 2024-09-21T00:20:05.414 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:05.414 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]: dispatch 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]: dispatch 2024-09-21T00:20:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]: dispatch 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:05.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:06.145 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 podman[92433]: 2024-09-21 00:20:05.693532 +0000 UTC m=+0.559307995 container remove 8906d6defaeb656431422a8835aff201631428c3a363faa8b2fcb9adfba5548e (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_CLEAN=True) 2024-09-21T00:20:06.145 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T00:20:06.145 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 systemd[1]: Stopped Ceph osd.1 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:20:06.145 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.1.service: Consumed 16.656s CPU time. 2024-09-21T00:20:06.145 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:05 smithi018 systemd[1]: Starting Ceph osd.1 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:20:06.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 podman[92532]: 2024-09-21 00:20:06.047324241 +0000 UTC m=+0.016704029 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:06.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 podman[92532]: 2024-09-21 00:20:06.181234867 +0000 UTC m=+0.150614708 container create 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:20:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: pgmap v79: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:20:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]': finished 2024-09-21T00:20:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-21T00:20:06.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[78227]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: pgmap v79: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]': finished 2024-09-21T00:20:06.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]': finished 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T00:20:06.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-mon[80555]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T00:20:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: pgmap v79: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:20:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 0]}]': finished 2024-09-21T00:20:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-09-21T00:20:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-09-21T00:20:06.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 5]}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.16", "id": [3, 1]}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [4, 1]}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 1]}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.7"}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.d"}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.10"}]': finished 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T00:20:06.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:06 smithi031 ceph-mon[63863]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 podman[92532]: 2024-09-21 00:20:06.456686988 +0000 UTC m=+0.426066766 container init 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=) 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 podman[92532]: 2024-09-21 00:20:06.460265339 +0000 UTC m=+0.429645123 container start 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 podman[92532]: 2024-09-21 00:20:06.555847329 +0000 UTC m=+0.525227104 container attach 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 bash[92532]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:06.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:06 smithi018 bash[92532]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:07.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T00:20:07.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T00:20:07.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T00:20:07.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate[92547]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 bash[92532]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T00:20:08.236 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:07 smithi018 podman[92532]: 2024-09-21 00:20:07.990751657 +0000 UTC m=+1.960131452 container died 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T00:20:08.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:08 smithi031 ceph-mon[63863]: pgmap v82: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:08.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:08 smithi031 ceph-mon[63863]: osdmap e124: 8 total, 7 up, 8 in 2024-09-21T00:20:08.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:08 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 98/723 objects degraded (13.555%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[78227]: pgmap v82: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[78227]: osdmap e124: 8 total, 7 up, 8 in 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 98/723 objects degraded (13.555%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[80555]: pgmap v82: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[80555]: osdmap e124: 8 total, 7 up, 8 in 2024-09-21T00:20:08.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 98/723 objects degraded (13.555%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:08.871 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 podman[92691]: 2024-09-21 00:20:08.526321379 +0000 UTC m=+0.529148718 container remove 119c3834656955181994d2f31bb4b65979da4a3943399fab15c0b06fc0cc23d3 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True) 2024-09-21T00:20:08.871 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 podman[92747]: 2024-09-21 00:20:08.646684008 +0000 UTC m=+0.016861821 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:08.872 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 podman[92747]: 2024-09-21 00:20:08.769067408 +0000 UTC m=+0.139245265 container create fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T00:20:09.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 podman[92747]: 2024-09-21 00:20:08.963628834 +0000 UTC m=+0.333806647 container init fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T00:20:09.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 podman[92747]: 2024-09-21 00:20:08.967041985 +0000 UTC m=+0.337219797 container start fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2) 2024-09-21T00:20:09.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:08 smithi018 ceph-osd[92765]: -- 172.21.15.18:0/3508524048 <== mon.1 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ac547b7a40 con 0x55ac5557e000 2024-09-21T00:20:09.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:09 smithi018 bash[92747]: fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 2024-09-21T00:20:09.425 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:09 smithi018 systemd[1]: Started Ceph osd.1 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:20:10.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[78227]: pgmap v84: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:10.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[80555]: pgmap v84: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:10 smithi031 ceph-mon[63863]: pgmap v84: 161 pgs: 6 peering, 42 active+undersized, 1 stale+active+clean, 18 active+undersized+degraded, 94 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%) 2024-09-21T00:20:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:10 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:10.645 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:10 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:20:10.385+0000 7faa75967640 -1 Falling back to public interface 2024-09-21T00:20:12.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:12 smithi018 ceph-mon[78227]: pgmap v85: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:12.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:12 smithi018 ceph-mon[80555]: pgmap v85: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:12.136 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:11] "GET /metrics HTTP/1.1" 200 38161 "" "Prometheus/2.43.0" 2024-09-21T00:20:12.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:12 smithi031 ceph-mon[63863]: pgmap v85: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:14.119 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:20:13.705+0000 7faa75967640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-21T00:20:14.119 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:20:13.742+0000 7faa75967640 -1 osd.1 119 log_to_monitors true 2024-09-21T00:20:14.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[78227]: pgmap v86: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[78227]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[80555]: pgmap v86: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:14 smithi018 ceph-mon[80555]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T00:20:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:14 smithi031 ceph-mon[63863]: pgmap v86: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:14.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:14 smithi031 ceph-mon[63863]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T00:20:15.056 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:20:15.016+0000 7faa6970a640 -1 osd.1 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: pgmap v87: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:20:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[80555]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: pgmap v87: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:20:15.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:15 smithi018 ceph-mon[78227]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: pgmap v87: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:20:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:15 smithi031 ceph-mon[63863]: from='osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241]' entity='osd.1' 2024-09-21T00:20:16.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[78227]: osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241] boot 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[78227]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[80555]: osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241] boot 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[80555]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T00:20:16.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:16 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:20:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:16 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:20:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:16 smithi031 ceph-mon[63863]: osd.1 [v2:172.21.15.18:6810/1565470241,v1:172.21.15.18:6811/1565470241] boot 2024-09-21T00:20:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:16 smithi031 ceph-mon[63863]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T00:20:16.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:16 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: pgmap v90: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:20:18.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: pgmap v90: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.430 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.431 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.432 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: pgmap v90: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%); 0 B/s, 0 objects/s recovering 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:18.504 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:18.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:18.505 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:19.330 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v92: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.330 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v93: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.330 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:20:19.330 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: Upgrade: Finalizing container_image settings 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: Upgrade: Complete! 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v94: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v95: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v96: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: pgmap v97: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v92: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v93: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: Upgrade: Finalizing container_image settings 2024-09-21T00:20:19.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: Upgrade: Complete! 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v94: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v95: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v96: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: pgmap v97: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T00:20:19.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:19.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v92: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v93: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:20:19.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:20:19.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v94: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v95: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v96: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: pgmap v97: 161 pgs: 9 active+clean+remapped, 42 active+undersized, 18 active+undersized+degraded, 92 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T00:20:19.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:20.417 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:21 smithi018 ceph-mon[78227]: pgmap v99: 161 pgs: 6 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 60/723 objects degraded (8.299%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:21 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:21 smithi018 ceph-mon[80555]: pgmap v99: 161 pgs: 6 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 60/723 objects degraded (8.299%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:21 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:21 smithi031 ceph-mon[63863]: pgmap v99: 161 pgs: 6 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 60/723 objects degraded (8.299%); 2/723 objects misplaced (0.277%) 2024-09-21T00:20:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:21 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:20:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:22 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded) 2024-09-21T00:20:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:22 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded) 2024-09-21T00:20:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:21] "GET /metrics HTTP/1.1" 200 38189 "" "Prometheus/2.43.0" 2024-09-21T00:20:22.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:22 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/723 objects degraded (8.299%), 12 pgs degraded) 2024-09-21T00:20:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:23 smithi018 ceph-mon[78227]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:23.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:23 smithi018 ceph-mon[80555]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:23 smithi031 ceph-mon[63863]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:25.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:25 smithi018 ceph-mon[78227]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:25.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:25 smithi018 ceph-mon[80555]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:25.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:25 smithi031 ceph-mon[63863]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:27.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:27 smithi018 ceph-mon[78227]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:27.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:27 smithi018 ceph-mon[80555]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:27.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:27 smithi031 ceph-mon[63863]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:27.968 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:20:28.286 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:28 smithi018 ceph-mon[78227]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:28.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:28 smithi018 ceph-mon[80555]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:28 smithi031 ceph-mon[63863]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:29 smithi018 ceph-mon[78227]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:29 smithi018 ceph-mon[80555]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:29 smithi031 ceph-mon[63863]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (17m) 16s ago 25m 23.7M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (4m) 2m ago 25m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 16s ago 24m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (4m) 2m ago 30m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (14m) 16s ago 31m 499M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (2m) 16s ago 31m 47.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (3m) 2m ago 30m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:20:29.983 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (2m) 16s ago 30m 38.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (17m) 16s ago 26m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (17m) 2m ago 26m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (56s) 16s ago 30m 43.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (21s) 16s ago 29m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (29m) 16s ago 29m 54.5M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (28m) 16s ago 28m 55.5M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (28m) 2m ago 28m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (28m) 2m ago 28m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (27m) 2m ago 27m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (27m) 2m ago 27m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (4m) 2m ago 26m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (24m) 16s ago 24m 80.0M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:20:29.984 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (24m) 2m ago 24m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:20:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:30 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:30.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:30 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:30.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:30 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:30.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-21T00:20:31.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:31 smithi018 ceph-mon[78227]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:31.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:31 smithi018 ceph-mon[78227]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:31.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:31 smithi018 ceph-mon[80555]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:31.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:31 smithi018 ceph-mon[80555]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:31 smithi031 ceph-mon[63863]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:31 smithi031 ceph-mon[63863]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:32.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:31] "GET /metrics HTTP/1.1" 200 38189 "" "Prometheus/2.43.0" 2024-09-21T00:20:32.863 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:20:33.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:33 smithi018 ceph-mon[78227]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:33.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:33 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2709802813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:33.167 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:33 smithi018 ceph-mon[80555]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:33.168 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:33 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2709802813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:33.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:33 smithi031 ceph-mon[63863]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T00:20:33.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:33 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2709802813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:33.552 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-09-21T00:20:35.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[78227]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:35.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:35.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:35.131 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[80555]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:35.132 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:35.132 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:35 smithi031 ceph-mon[63863]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:35.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:35.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:36.898 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:20:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:37 smithi018 ceph-mon[78227]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:37 smithi018 ceph-mon[78227]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:37 smithi018 ceph-mon[80555]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:37 smithi018 ceph-mon[80555]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:37 smithi031 ceph-mon[63863]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:37 smithi031 ceph-mon[63863]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:37.535 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:20:38.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:38 smithi018 ceph-mon[78227]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:38.923 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:38 smithi018 ceph-mon[80555]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:38 smithi031 ceph-mon[63863]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:39.677 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:20:39.678 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:20:39.678 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:20:39.678 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:20:39.679 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:20:39.679 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:20:39.680 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:20:39.680 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:20:39.680 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:20:40.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:39 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:40.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:39 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:39 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:40.363 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:20:40.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:40 smithi018 ceph-mon[78227]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:40.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:40 smithi018 ceph-mon[78227]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:40.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:40 smithi018 ceph-mon[80555]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:40.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:40 smithi018 ceph-mon[80555]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:40 smithi031 ceph-mon[63863]: from='client.54247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:40 smithi031 ceph-mon[63863]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:41] "GET /metrics HTTP/1.1" 200 38184 "" "Prometheus/2.43.0" 2024-09-21T00:20:42.392 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:20:42.392 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:20:42.392 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:20:42.998 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-09-21T00:20:43.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:43 smithi018 ceph-mon[78227]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:43.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:43 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2635072691' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:20:43.304 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:43 smithi018 ceph-mon[80555]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:43.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:43 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2635072691' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:20:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:43 smithi031 ceph-mon[63863]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:43 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2635072691' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:20:45.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:45 smithi018 ceph-mon[78227]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:45 smithi018 ceph-mon[80555]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:45 smithi031 ceph-mon[63863]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:20:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:46 smithi018 ceph-mon[78227]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:46 smithi018 ceph-mon[80555]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:46.426 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:46 smithi031 ceph-mon[63863]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:47.055 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:20:47.376 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[80555]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:47.376 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.376 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.376 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[78227]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:47.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:47 smithi031 ceph-mon[63863]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:20:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:48.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:48.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:20:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-09-21T00:20:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:49.026 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:20:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:49 smithi018 ceph-mon[80555]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:49 smithi018 ceph-mon[78227]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:49 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:49.735 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (18m) 35s ago 26m 23.7M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (4m) 2m ago 25m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 35s ago 25m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (4m) 2m ago 30m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (14m) 35s ago 32m 499M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (2m) 35s ago 32m 47.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (3m) 2m ago 30m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (2m) 35s ago 31m 38.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (18m) 35s ago 26m 16.5M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (17m) 2m ago 26m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (76s) 35s ago 30m 43.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:20:49.736 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (41s) 35s ago 29m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (29m) 35s ago 29m 54.5M 2993M 17.2.0 e1d6a67b021e 3b9c01d2567a 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (29m) 35s ago 29m 55.5M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (28m) 2m ago 28m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (28m) 2m ago 28m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (27m) 2m ago 27m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (27m) 2m ago 27m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (5m) 2m ago 26m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (25m) 35s ago 25m 80.0M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:20:49.737 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (25m) 2m ago 25m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:20:50.193 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 7 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:20:50.194 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:20:50.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3275203931' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:50.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:50 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3275203931' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='client.44328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:50 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3275203931' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "2/8 daemons upgraded", 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:20:50.577 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:20:51.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:20:51.225 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:20:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:20:51.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:51.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: Upgrade: osd.2 is safe to restart 2024-09-21T00:20:51.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:51.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:20:51.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:51.557 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mgr 2024-09-21T00:20:51.557 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: Upgrade: osd.2 is safe to restart 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:51.558 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mgr 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: Upgrade: osd.2 is safe to restart 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T00:20:51.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:20:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:20:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:20:51] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-09-21T00:20:52.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[80555]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[80555]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[80555]: Upgrade: Updating osd.2 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[80555]: Deploying daemon osd.2 on smithi018 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[78227]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[78227]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[78227]: Upgrade: Updating osd.2 2024-09-21T00:20:52.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:52 smithi018 ceph-mon[78227]: Deploying daemon osd.2 on smithi018 2024-09-21T00:20:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:52 smithi031 ceph-mon[63863]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:20:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:52 smithi031 ceph-mon[63863]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:52 smithi031 ceph-mon[63863]: Upgrade: Updating osd.2 2024-09-21T00:20:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:52 smithi031 ceph-mon[63863]: Deploying daemon osd.2 on smithi018 2024-09-21T00:20:54.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:54 smithi018 systemd[1]: Stopping Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:20:54.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:54 smithi031 ceph-mon[63863]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:54 smithi018 ceph-mon[80555]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:54 smithi018 ceph-mon[78227]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:20:54.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[37014]: 2024-09-21T00:20:54.374+0000 7f58457c1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:20:54.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[37014]: 2024-09-21T00:20:54.374+0000 7f58457c1700 -1 osd.2 128 *** Got signal Terminated *** 2024-09-21T00:20:54.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[37014]: 2024-09-21T00:20:54.374+0000 7f58457c1700 -1 osd.2 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:20:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:55 smithi031 ceph-mon[63863]: osd.2 marked itself down and dead 2024-09-21T00:20:55.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:55 smithi018 ceph-mon[78227]: osd.2 marked itself down and dead 2024-09-21T00:20:55.657 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:55 smithi018 ceph-mon[80555]: osd.2 marked itself down and dead 2024-09-21T00:20:55.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:55 smithi018 podman[98796]: 2024-09-21 00:20:55.655459164 +0000 UTC m=+1.380451192 container died 3b9c01d2567aa2a7c2270c9146072c6d893366ef9bb21b91774d12545c972dbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-type=git, RELEASE=HEAD, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T00:20:56.244 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98796]: 2024-09-21 00:20:56.016486009 +0000 UTC m=+1.741478049 container cleanup 3b9c01d2567aa2a7c2270c9146072c6d893366ef9bb21b91774d12545c972dbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T00:20:56.245 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 bash[98796]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2 2024-09-21T00:20:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:56 smithi031 ceph-mon[63863]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-09-21T00:20:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:56 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:56 smithi031 ceph-mon[63863]: osdmap e129: 8 total, 7 up, 8 in 2024-09-21T00:20:56.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[78227]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-09-21T00:20:56.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:56.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[78227]: osdmap e129: 8 total, 7 up, 8 in 2024-09-21T00:20:56.650 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[80555]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-09-21T00:20:56.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:20:56.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:56 smithi018 ceph-mon[80555]: osdmap e129: 8 total, 7 up, 8 in 2024-09-21T00:20:56.651 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98814]: 2024-09-21 00:20:56.301153327 +0000 UTC m=+0.636326841 container remove 3b9c01d2567aa2a7c2270c9146072c6d893366ef9bb21b91774d12545c972dbf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-21T00:20:56.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98901]: 2024-09-21 00:20:56.55267923 +0000 UTC m=+0.016699412 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:56.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98901]: 2024-09-21 00:20:56.685442653 +0000 UTC m=+0.149462824 container create 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T00:20:57.249 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98901]: 2024-09-21 00:20:56.930048756 +0000 UTC m=+0.394068936 container init 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:20:57.249 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:56 smithi018 podman[98901]: 2024-09-21 00:20:56.933616335 +0000 UTC m=+0.397636515 container start 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70) 2024-09-21T00:20:57.249 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 podman[98901]: 2024-09-21 00:20:57.021782318 +0000 UTC m=+0.485802505 container attach 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T00:20:57.249 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 podman[98901]: 2024-09-21 00:20:57.127643856 +0000 UTC m=+0.591664037 container died 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T00:20:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:57 smithi031 ceph-mon[63863]: osdmap e130: 8 total, 7 up, 8 in 2024-09-21T00:20:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:57 smithi018 ceph-mon[78227]: osdmap e130: 8 total, 7 up, 8 in 2024-09-21T00:20:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:57 smithi018 ceph-mon[80555]: osdmap e130: 8 total, 7 up, 8 in 2024-09-21T00:20:58.136 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 podman[98924]: 2024-09-21 00:20:57.67549966 +0000 UTC m=+0.541595521 container remove 0c4406f2e0782f9df18eda8ba7e8e64e2aff05c8d89c158d5b3039ef0c37b656 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T00:20:58.136 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T00:20:58.136 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 systemd[1]: Stopped Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:20:58.137 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service: Consumed 14.242s CPU time. 2024-09-21T00:20:58.137 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:57 smithi018 systemd[1]: Starting Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:20:58.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-mon[78227]: pgmap v121: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-21T00:20:58.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-mon[78227]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T00:20:58.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-mon[80555]: pgmap v121: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-21T00:20:58.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-mon[80555]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T00:20:58.425 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 podman[99022]: 2024-09-21 00:20:58.038609666 +0000 UTC m=+0.016811549 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:20:58.425 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 podman[99022]: 2024-09-21 00:20:58.157743876 +0000 UTC m=+0.135945759 container create 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T00:20:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:58 smithi031 ceph-mon[63863]: pgmap v121: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-21T00:20:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:20:58 smithi031 ceph-mon[63863]: Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T00:20:58.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 podman[99022]: 2024-09-21 00:20:58.430295246 +0000 UTC m=+0.408497138 container init 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T00:20:58.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 podman[99022]: 2024-09-21 00:20:58.434119785 +0000 UTC m=+0.412321671 container start 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:20:58.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 podman[99022]: 2024-09-21 00:20:58.51342469 +0000 UTC m=+0.491626578 container attach 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T00:20:58.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:58.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 bash[99022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:58.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:58.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:58 smithi018 bash[99022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:20:59.643 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T00:20:59.644 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T00:20:59.644 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T00:20:59.644 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T00:20:59.644 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T00:20:59.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T00:20:59.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T00:20:59.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate[99037]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 bash[99022]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T00:20:59.926 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:20:59 smithi018 podman[99022]: 2024-09-21 00:20:59.674560489 +0000 UTC m=+1.652762372 container died 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:21:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:00 smithi018 ceph-mon[78227]: pgmap v122: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:21:00.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:00 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:00.565 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:00 smithi018 ceph-mon[80555]: pgmap v122: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:21:00.565 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:00 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:00.566 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 podman[99170]: 2024-09-21 00:21:00.267347563 +0000 UTC m=+0.582629540 container remove 6861886cba20f06bb3d45f1d8d58ffe7c9d828e9c8240383bf44adbb230b4497 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T00:21:00.566 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 podman[99228]: 2024-09-21 00:21:00.47672395 +0000 UTC m=+0.107124492 container create 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T00:21:00.566 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 podman[99228]: 2024-09-21 00:21:00.386478898 +0000 UTC m=+0.016879408 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:00 smithi031 ceph-mon[63863]: pgmap v122: 161 pgs: 31 peering, 130 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:21:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:00 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:00.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 podman[99228]: 2024-09-21 00:21:00.712636446 +0000 UTC m=+0.343036956 container init 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:21:00.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 podman[99228]: 2024-09-21 00:21:00.715956359 +0000 UTC m=+0.346356860 container start 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T00:21:00.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 ceph-osd[99247]: -- 172.21.15.18:0/3423381460 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556f8a627a40 con 0x556f8a7de000 2024-09-21T00:21:00.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 bash[99228]: 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a 2024-09-21T00:21:00.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:00 smithi018 systemd[1]: Started Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:21:02.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[78227]: pgmap v123: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:01] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:01 smithi018 ceph-mon[80555]: pgmap v123: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:01 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:01 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:01 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:02.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:01 smithi031 ceph-mon[63863]: pgmap v123: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:02.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:02 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 65/723 objects degraded (8.990%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:02.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:02 smithi018 ceph-mon[80555]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T00:21:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:02 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 65/723 objects degraded (8.990%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:02 smithi018 ceph-mon[78227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T00:21:02.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:02 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:21:02.605+0000 7f291dfb5640 -1 Falling back to public interface 2024-09-21T00:21:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:02 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 65/723 objects degraded (8.990%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:02 smithi031 ceph-mon[63863]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-21T00:21:04.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:03 smithi018 ceph-mon[78227]: pgmap v124: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:04.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:03 smithi018 ceph-mon[80555]: pgmap v124: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:03 smithi031 ceph-mon[63863]: pgmap v124: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:05.674 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[78227]: pgmap v125: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 103 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:05 smithi018 ceph-mon[80555]: pgmap v125: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 103 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:05.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:05.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:05.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:05 smithi031 ceph-mon[63863]: pgmap v125: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 143 MiB used, 715 GiB / 715 GiB avail; 103 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:06.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:21:06.208+0000 7f291dfb5640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-21T00:21:06.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:21:06.227+0000 7f291dfb5640 -1 osd.2 128 log_to_monitors true 2024-09-21T00:21:06.963 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:21:06.708+0000 7f2915d60640 -1 osd.2 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.964 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.965 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.965 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:06.965 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:06.965 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:06.965 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T00:21:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:07.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:07 smithi031 ceph-mon[63863]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T00:21:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:07 smithi031 ceph-mon[63863]: osdmap e131: 8 total, 7 up, 8 in 2024-09-21T00:21:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:07 smithi031 ceph-mon[63863]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:07 smithi031 ceph-mon[63863]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:07 smithi031 ceph-mon[63863]: pgmap v127: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:08.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[78227]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T00:21:08.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[78227]: osdmap e131: 8 total, 7 up, 8 in 2024-09-21T00:21:08.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[78227]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[78227]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[78227]: pgmap v127: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[80555]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[80555]: osdmap e131: 8 total, 7 up, 8 in 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[80555]: from='osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[80555]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:21:08.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:07 smithi018 ceph-mon[80555]: pgmap v127: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:08 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:21:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:08 smithi031 ceph-mon[63863]: osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649] boot 2024-09-21T00:21:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:08 smithi031 ceph-mon[63863]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T00:21:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:08 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:21:09.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:21:09.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[78227]: osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649] boot 2024-09-21T00:21:09.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[78227]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T00:21:09.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:21:09.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:21:09.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[80555]: osd.2 [v2:172.21.15.18:6818/3361091649,v1:172.21.15.18:6819/3361091649] boot 2024-09-21T00:21:09.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[80555]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T00:21:09.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:08 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: pgmap v130: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: pgmap v130: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:09.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:10.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T00:21:10.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: pgmap v130: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:10.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v131: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v132: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:21:11.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.004 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v133: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v134: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v135: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: pgmap v136: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.005 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:10 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v131: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v132: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: Upgrade: Finalizing container_image settings 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:21:11.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v131: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v132: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: Upgrade: Finalizing container_image settings 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:21:11.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: Upgrade: Complete! 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v133: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v134: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v135: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: pgmap v136: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: Upgrade: Complete! 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v133: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v134: 161 pgs: 32 active+undersized, 14 active+undersized+degraded, 115 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:11.182 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v135: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: pgmap v136: 161 pgs: 28 active+undersized, 12 active+undersized+degraded, 121 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:11.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:10 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:12.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:11 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:11 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:11 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:11] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-09-21T00:21:13.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:12 smithi031 ceph-mon[63863]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-09-21T00:21:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:12 smithi018 ceph-mon[78227]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-09-21T00:21:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:12 smithi018 ceph-mon[80555]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-09-21T00:21:14.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:13 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded) 2024-09-21T00:21:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:13 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded) 2024-09-21T00:21:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:13 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded) 2024-09-21T00:21:15.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:14 smithi031 ceph-mon[63863]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:15.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:14 smithi018 ceph-mon[78227]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:14 smithi018 ceph-mon[80555]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:14 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:16 smithi018 ceph-mon[78227]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-21T00:21:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:16 smithi018 ceph-mon[80555]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-21T00:21:17.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:16 smithi031 ceph-mon[63863]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-21T00:21:19.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:18 smithi018 ceph-mon[78227]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:19.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:18 smithi018 ceph-mon[80555]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:19.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:18 smithi031 ceph-mon[63863]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:19 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:20.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:20 smithi031 ceph-mon[63863]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:21:21.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:20 smithi018 ceph-mon[78227]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:21:21.329 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:20 smithi018 ceph-mon[80555]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:21:21.615 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:21:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:21 smithi018 ceph-mon[78227]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:21 smithi018 ceph-mon[80555]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:21] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-09-21T00:21:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:21 smithi031 ceph-mon[63863]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:23.155 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:22 smithi018 ceph-mon[80555]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:23.156 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:22 smithi018 ceph-mon[78227]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:23.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:22 smithi031 ceph-mon[63863]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (18m) 17s ago 26m 23.8M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (5m) 3m ago 26m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 17s ago 25m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (5m) 3m ago 31m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (15m) 17s ago 32m 502M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (3m) 17s ago 32m 53.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (4m) 3m ago 31m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (3m) 17s ago 31m 40.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (18m) 17s ago 27m 16.8M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:21:23.614 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (18m) 3m ago 27m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (110s) 17s ago 30m 44.2M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (74s) 17s ago 30m 51.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (23s) 17s ago 30m 12.4M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (29m) 17s ago 29m 56.2M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (29m) 3m ago 29m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (28m) 3m ago 28m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (28m) 3m ago 28m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (28m) 3m ago 28m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (5m) 3m ago 27m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (25m) 17s ago 25m 80.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:21:23.615 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (25m) 3m ago 25m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:21:24.254 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-21T00:21:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:24 smithi018 ceph-mon[78227]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:24 smithi018 ceph-mon[78227]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:24 smithi018 ceph-mon[80555]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:24 smithi018 ceph-mon[80555]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:24 smithi031 ceph-mon[63863]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:24 smithi031 ceph-mon[63863]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:26.301 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:21:26.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-09-21T00:21:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:26 smithi031 ceph-mon[63863]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:26 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/993632507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:27.276 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:26 smithi018 ceph-mon[80555]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:27.276 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:26 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/993632507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:27.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:26 smithi018 ceph-mon[78227]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:27.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:26 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/993632507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:29.241 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:28 smithi018 ceph-mon[78227]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:29.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:28 smithi018 ceph-mon[80555]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:28 smithi031 ceph-mon[63863]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:29 smithi031 ceph-mon[63863]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:30.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:29 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:30.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:29 smithi018 ceph-mon[78227]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:30.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:29 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:30.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:29 smithi018 ceph-mon[80555]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:30.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:29 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:30.431 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:21:31.023 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:30 smithi018 ceph-mon[78227]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:31.023 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:30 smithi018 ceph-mon[80555]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:31.051 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:21:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:30 smithi031 ceph-mon[63863]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:32.043 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:31] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-09-21T00:21:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:32 smithi018 ceph-mon[78227]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:32 smithi018 ceph-mon[80555]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:33.192 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:21:33.192 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:21:33.192 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:21:33.192 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:21:33.192 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:21:33.193 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:21:33.193 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:21:33.193 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:21:33.193 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:21:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:32 smithi031 ceph-mon[63863]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:33.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:21:34.203 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:33 smithi018 ceph-mon[78227]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:34.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:33 smithi018 ceph-mon[80555]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:33 smithi031 ceph-mon[63863]: from='client.44388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:35.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:34 smithi018 ceph-mon[78227]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:35.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:35.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:34 smithi018 ceph-mon[80555]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:35.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:34 smithi031 ceph-mon[63863]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:35.766 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:21:35.766 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:21:35.767 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:21:36.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:35 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3537465872' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:21:36.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:35 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3537465872' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:21:36.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:35 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3537465872' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:21:36.417 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-09-21T00:21:37.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:36 smithi018 ceph-mon[80555]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:37.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:36 smithi018 ceph-mon[78227]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:36 smithi031 ceph-mon[63863]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:39.244 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:38 smithi018 ceph-mon[78227]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:39.244 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:38 smithi018 ceph-mon[80555]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:38 smithi031 ceph-mon[63863]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:39.976 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:39 smithi031 ceph-mon[63863]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:39 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:40.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:39 smithi018 ceph-mon[80555]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:40.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:39 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:40.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:39 smithi018 ceph-mon[78227]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:40.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:39 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:40.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:21:41.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:41.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[80555]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-09-21T00:21:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:40 smithi018 ceph-mon[78227]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:41.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:40 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:42.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:41] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-09-21T00:21:43.297 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:43 smithi018 ceph-mon[80555]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:43.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:43 smithi018 ceph-mon[78227]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:43.458 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:21:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:43 smithi031 ceph-mon[63863]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:44.172 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (18m) 38s ago 26m 23.8M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (5m) 3m ago 26m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 38s ago 25m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (5m) 3m ago 31m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (15m) 38s ago 33m 502M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (3m) 38s ago 33m 53.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (4m) 3m ago 31m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (3m) 38s ago 31m 40.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (19m) 38s ago 27m 16.8M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (18m) 3m ago 27m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (2m) 38s ago 31m 44.2M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (95s) 38s ago 30m 51.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:21:44.173 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (43s) 38s ago 30m 12.4M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (30m) 38s ago 30m 56.2M 2993M 17.2.0 e1d6a67b021e 6d923b3ce16d 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (29m) 3m ago 29m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (29m) 3m ago 29m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (28m) 3m ago 28m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (28m) 3m ago 28m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (5m) 3m ago 27m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (26m) 38s ago 26m 80.3M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:21:44.174 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (26m) 3m ago 26m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:21:44.593 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:21:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:21:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:21:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:21:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T00:21:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-21T00:21:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:21:44.595 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mgr 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:21:44.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: Upgrade: osd.3 is safe to restart 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2759130624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mgr 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:44.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:44.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: Upgrade: osd.3 is safe to restart 2024-09-21T00:21:44.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:44.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:44.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:44 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2759130624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "3/8 daemons upgraded", 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:21:44.948 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:21:44.949 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:21:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='client.44406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:21:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:21:45.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: Upgrade: osd.3 is safe to restart 2024-09-21T00:21:45.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-21T00:21:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:45.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:44 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2759130624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:21:46.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: Upgrade: Updating osd.3 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[78227]: Deploying daemon osd.3 on smithi018 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: Upgrade: Updating osd.3 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:21:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:46 smithi018 ceph-mon[80555]: Deploying daemon osd.3 on smithi018 2024-09-21T00:21:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:21:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: Upgrade: Updating osd.3 2024-09-21T00:21:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T00:21:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:21:46.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:46 smithi031 ceph-mon[63863]: Deploying daemon osd.3 on smithi018 2024-09-21T00:21:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:47 smithi018 ceph-mon[78227]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:47 smithi018 ceph-mon[80555]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:47 smithi031 ceph-mon[63863]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:48.568 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:48 smithi018 systemd[1]: Stopping Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:21:48.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[39980]: 2024-09-21T00:21:48.566+0000 7f794ffbb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:21:48.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[39980]: 2024-09-21T00:21:48.566+0000 7f794ffbb700 -1 osd.3 133 *** Got signal Terminated *** 2024-09-21T00:21:48.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:48 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[39980]: 2024-09-21T00:21:48.566+0000 7f794ffbb700 -1 osd.3 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:21:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:49 smithi018 ceph-mon[78227]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:49 smithi018 ceph-mon[78227]: osd.3 marked itself down and dead 2024-09-21T00:21:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:49 smithi018 ceph-mon[80555]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:49 smithi018 ceph-mon[80555]: osd.3 marked itself down and dead 2024-09-21T00:21:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:49 smithi031 ceph-mon[63863]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:21:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:49 smithi031 ceph-mon[63863]: osd.3 marked itself down and dead 2024-09-21T00:21:49.749 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:49 smithi018 podman[105299]: 2024-09-21 00:21:49.549925461 +0000 UTC m=+1.114775919 container died 6d923b3ce16d80aebe966b39def3f76e366e610ad05ee36d851e1f0d98da711d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, name=centos-stream, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T00:21:50.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:21:50.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[80555]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T00:21:50.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[78227]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:50.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:50.397 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 podman[105299]: 2024-09-21 00:21:50.002150498 +0000 UTC m=+1.567000934 container cleanup 6d923b3ce16d80aebe966b39def3f76e366e610ad05ee36d851e1f0d98da711d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8) 2024-09-21T00:21:50.398 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 bash[105299]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3 2024-09-21T00:21:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:50 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:21:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:50 smithi031 ceph-mon[63863]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T00:21:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:50 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:21:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:21:50.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 podman[105315]: 2024-09-21 00:21:50.395868407 +0000 UTC m=+0.847876101 container remove 6d923b3ce16d80aebe966b39def3f76e366e610ad05ee36d851e1f0d98da711d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-09-21T00:21:51.100 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 podman[105402]: 2024-09-21 00:21:50.677223554 +0000 UTC m=+0.016253447 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:51.100 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 podman[105402]: 2024-09-21 00:21:50.779956167 +0000 UTC m=+0.118986037 container create bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T00:21:51.101 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:50 smithi018 podman[105402]: 2024-09-21 00:21:50.999051112 +0000 UTC m=+0.338080994 container init bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T00:21:51.101 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 podman[105402]: 2024-09-21 00:21:51.003187986 +0000 UTC m=+0.342217861 container start bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T00:21:51.101 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 podman[105402]: 2024-09-21 00:21:51.099936204 +0000 UTC m=+0.438966079 container attach bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T00:21:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[78227]: pgmap v159: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-21T00:21:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[78227]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T00:21:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[78227]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T00:21:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[80555]: pgmap v159: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-21T00:21:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[80555]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T00:21:51.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-mon[80555]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T00:21:51.426 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 podman[105402]: 2024-09-21 00:21:51.200903638 +0000 UTC m=+0.539933525 container died bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T00:21:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:51 smithi031 ceph-mon[63863]: pgmap v159: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-21T00:21:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:51 smithi031 ceph-mon[63863]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T00:21:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:51 smithi031 ceph-mon[63863]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T00:21:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:21:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:21:51] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-09-21T00:21:52.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 podman[105430]: 2024-09-21 00:21:51.802758225 +0000 UTC m=+0.595680300 container remove bf356ec4ddbf881c7441ace37642c23e1268256bdff83749c9197a360514962b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T00:21:52.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T00:21:52.176 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 systemd[1]: Stopped Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:21:52.176 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:51 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service: Consumed 18.155s CPU time. 2024-09-21T00:21:52.176 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 systemd[1]: Starting Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:21:52.499 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:52 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:52.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:52 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:52.559 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 podman[105535]: 2024-09-21 00:21:52.149159873 +0000 UTC m=+0.017375315 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:52.559 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 podman[105535]: 2024-09-21 00:21:52.279137006 +0000 UTC m=+0.147352474 container create d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T00:21:52.559 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 podman[105535]: 2024-09-21 00:21:52.498159793 +0000 UTC m=+0.366375228 container init d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T00:21:52.560 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:52 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 43/723 objects degraded (5.947%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:52.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 podman[105535]: 2024-09-21 00:21:52.501837646 +0000 UTC m=+0.370053078 container start d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T00:21:52.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 podman[105535]: 2024-09-21 00:21:52.589783758 +0000 UTC m=+0.457999200 container attach d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:21:52.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:21:52.926 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 bash[105535]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:21:52.926 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:21:52.926 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:52 smithi018 bash[105535]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:21:53.406 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-mon[78227]: pgmap v161: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:53.406 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-mon[80555]: pgmap v161: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:53.407 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T00:21:53.407 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T00:21:53.407 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T00:21:53.407 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T00:21:53.407 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T00:21:53.408 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T00:21:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:53 smithi031 ceph-mon[63863]: pgmap v161: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T00:21:53.675 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T00:21:53.676 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate[105550]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T00:21:53.676 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 bash[105535]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T00:21:53.676 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:53 smithi018 podman[105535]: 2024-09-21 00:21:53.434144076 +0000 UTC m=+1.302359515 container died d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T00:21:54.422 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 podman[105687]: 2024-09-21 00:21:54.085429673 +0000 UTC m=+0.645066206 container remove d8337ec7d27131489d67d046ea0e4723b665471dc913bd75fe98269643cbaa5c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T00:21:54.422 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 podman[105746]: 2024-09-21 00:21:54.202401035 +0000 UTC m=+0.017169356 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:21:54.422 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 podman[105746]: 2024-09-21 00:21:54.311279047 +0000 UTC m=+0.126047374 container create 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:21:54.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 podman[105746]: 2024-09-21 00:21:54.538702032 +0000 UTC m=+0.353470359 container init 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T00:21:54.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 podman[105746]: 2024-09-21 00:21:54.542073879 +0000 UTC m=+0.356842201 container start 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70) 2024-09-21T00:21:54.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 ceph-osd[105765]: -- 172.21.15.18:0/1065740642 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559d56b6da40 con 0x559d57934400 2024-09-21T00:21:54.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 bash[105746]: 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 2024-09-21T00:21:54.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:54 smithi018 systemd[1]: Started Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:21:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:55 smithi031 ceph-mon[63863]: pgmap v162: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[78227]: pgmap v162: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[80555]: pgmap v162: 161 pgs: 23 active+undersized, 26 peering, 12 active+undersized+degraded, 100 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 43/723 objects degraded (5.947%) 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:55.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:21:56.243 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:21:55.954+0000 7f16206f6640 -1 Falling back to public interface 2024-09-21T00:21:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:56 smithi031 ceph-mon[63863]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 12 pgs peering) 2024-09-21T00:21:56.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:56 smithi018 ceph-mon[80555]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 12 pgs peering) 2024-09-21T00:21:56.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:56 smithi018 ceph-mon[78227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 12 pgs peering) 2024-09-21T00:21:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:57 smithi031 ceph-mon[63863]: pgmap v163: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:57.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:57 smithi018 ceph-mon[78227]: pgmap v163: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:57.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:57 smithi018 ceph-mon[80555]: pgmap v163: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:58 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 94/723 objects degraded (13.001%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:58.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:58 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 94/723 objects degraded (13.001%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:58.523 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:58 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 94/723 objects degraded (13.001%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T00:21:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:21:59 smithi018 ceph-mon[78227]: pgmap v164: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:21:59 smithi018 ceph-mon[80555]: pgmap v164: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:59.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:21:59 smithi031 ceph-mon[63863]: pgmap v164: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:21:59.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:21:59.489+0000 7f16206f6640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-21T00:21:59.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:21:59 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:21:59.521+0000 7f16206f6640 -1 osd.3 133 log_to_monitors true 2024-09-21T00:22:00.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T00:22:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:00.838 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:22:00 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:22:00.784+0000 7f1614499640 -1 osd.3 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: pgmap v165: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.614 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[80555]: from='osd.3 ' entity='osd.3' 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: pgmap v165: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-mon[78227]: from='osd.3 ' entity='osd.3' 2024-09-21T00:22:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: pgmap v165: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 160 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-09-21T00:22:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T00:22:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T00:22:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: from='osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T00:22:01.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:01 smithi031 ceph-mon[63863]: from='osd.3 ' entity='osd.3' 2024-09-21T00:22:02.154 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:01] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[80555]: osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790] boot 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[80555]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[78227]: osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790] boot 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[78227]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T00:22:02.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:22:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:02 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:02 smithi031 ceph-mon[63863]: osd.3 [v2:172.21.15.18:6826/1805328790,v1:172.21.15.18:6827/1805328790] boot 2024-09-21T00:22:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:02 smithi031 ceph-mon[63863]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T00:22:02.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T00:22:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:03 smithi018 ceph-mon[78227]: pgmap v168: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:03 smithi018 ceph-mon[78227]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T00:22:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:03 smithi018 ceph-mon[80555]: pgmap v168: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:03 smithi018 ceph-mon[80555]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T00:22:03.652 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:03 smithi031 ceph-mon[63863]: pgmap v168: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:03.652 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:03 smithi031 ceph-mon[63863]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T00:22:03.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:03 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:22:03.698+0000 7fd0517a1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-21T00:22:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: pgmap v170: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 916 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: pgmap v171: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: pgmap v170: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 916 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: pgmap v171: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:04.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:04.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:04.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-21T00:22:04.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:04.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: pgmap v170: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 916 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: pgmap v171: 161 pgs: 39 active+undersized, 22 active+undersized+degraded, 100 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 94/723 objects degraded (13.001%) 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-21T00:22:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:05.003 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:05 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:22:06.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:05 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:22:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:05 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:22:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:06 smithi031 ceph-mon[63863]: pgmap v172: 161 pgs: 4 active+undersized, 1 active+undersized+degraded, 156 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-21T00:22:07.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:06 smithi018 ceph-mon[78227]: pgmap v172: 161 pgs: 4 active+undersized, 1 active+undersized+degraded, 156 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-21T00:22:07.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:06 smithi018 ceph-mon[80555]: pgmap v172: 161 pgs: 4 active+undersized, 1 active+undersized+degraded, 156 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-21T00:22:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:07 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-21T00:22:08.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:07 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-21T00:22:08.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:07 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-21T00:22:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:08 smithi031 ceph-mon[63863]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:09.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:08 smithi018 ceph-mon[78227]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:09.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:08 smithi018 ceph-mon[80555]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:10.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:09 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:10.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:09 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:09 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:11.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:10 smithi031 ceph-mon[63863]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:10 smithi018 ceph-mon[78227]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:10 smithi018 ceph-mon[80555]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:11] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.43.0" 2024-09-21T00:22:13.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:12 smithi031 ceph-mon[63863]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:12 smithi018 ceph-mon[78227]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:12 smithi018 ceph-mon[80555]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:15.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:14 smithi031 ceph-mon[63863]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:15.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:14 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:15.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:14 smithi018 ceph-mon[80555]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:14 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:14 smithi018 ceph-mon[78227]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:22:15.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:14 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:15.329 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:22:16.015 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:22:16.015 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (19m) 16s ago 27m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (6m) 4m ago 27m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 16s ago 26m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (6m) 4m ago 32m 437M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (16m) 16s ago 33m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (4m) 16s ago 33m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (5m) 4m ago 32m 35.8M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (3m) 16s ago 32m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (19m) 16s ago 28m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (19m) 4m ago 28m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (2m) 16s ago 31m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (2m) 16s ago 31m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (75s) 16s ago 30m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (21s) 16s ago 30m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:22:16.016 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (30m) 4m ago 30m 57.3M 3249M 17.2.0 e1d6a67b021e fab2520e6b9f 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (29m) 4m ago 29m 52.2M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (29m) 4m ago 29m 53.6M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (28m) 4m ago 28m 56.2M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (6m) 4m ago 28m 64.7M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (26m) 16s ago 26m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:22:16.017 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (26m) 4m ago 26m 80.5M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:22:16.433 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 4 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:22:16.434 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 9 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:22:16.435 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:22:16.786 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "4/8 daemons upgraded", 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:22:16.787 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:22:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:16 smithi031 ceph-mon[63863]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:16 smithi031 ceph-mon[63863]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T00:22:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:16 smithi031 ceph-mon[63863]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:16 smithi031 ceph-mon[63863]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:16 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1980264266' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:17.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[80555]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[80555]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[80555]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[80555]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1980264266' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[78227]: from='client.54376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[78227]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[78227]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[78227]: from='client.54388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:17.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:16 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1980264266' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:18.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:17 smithi018 ceph-mon[78227]: from='client.54400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:18.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:17 smithi018 ceph-mon[80555]: from='client.54400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:18.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:17 smithi031 ceph-mon[63863]: from='client.54400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:19.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:18 smithi018 ceph-mon[78227]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:19.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:18 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:18 smithi018 ceph-mon[80555]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:19.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:18 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:19.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:18 smithi031 ceph-mon[63863]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:19.109 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:18 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:20.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:19 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:20.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:19 smithi031 ceph-mon[63863]: Upgrade: osd.4 is safe to restart 2024-09-21T00:22:20.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:20.133 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[78227]: Upgrade: osd.4 is safe to restart 2024-09-21T00:22:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:20.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T00:22:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[80555]: Upgrade: osd.4 is safe to restart 2024-09-21T00:22:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:20.996 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:20.997 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:20 smithi031 ceph-mon[63863]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:20.997 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:20.997 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:22:20.997 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[78227]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:22:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[80555]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T00:22:21.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:21.888 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:21 smithi031 ceph-mon[63863]: Upgrade: Updating osd.4 2024-09-21T00:22:21.888 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:21 smithi031 ceph-mon[63863]: Deploying daemon osd.4 on smithi031 2024-09-21T00:22:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:21] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-09-21T00:22:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:21 smithi018 ceph-mon[78227]: Upgrade: Updating osd.4 2024-09-21T00:22:22.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:21 smithi018 ceph-mon[78227]: Deploying daemon osd.4 on smithi031 2024-09-21T00:22:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:21 smithi018 ceph-mon[80555]: Upgrade: Updating osd.4 2024-09-21T00:22:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:21 smithi018 ceph-mon[80555]: Deploying daemon osd.4 on smithi031 2024-09-21T00:22:22.890 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:22 smithi031 ceph-mon[63863]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:23.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:22 smithi018 ceph-mon[78227]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:23.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:22 smithi018 ceph-mon[80555]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:23.579 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:23 smithi031 systemd[1]: Stopping Ceph osd.4 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:22:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:23 smithi031 ceph-mon[63863]: osd.4 marked itself down and dead 2024-09-21T00:22:24.001 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[27828]: 2024-09-21T00:22:23.577+0000 7f04ac4ed700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:22:24.001 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[27828]: 2024-09-21T00:22:23.577+0000 7f04ac4ed700 -1 osd.4 138 *** Got signal Terminated *** 2024-09-21T00:22:24.001 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[27828]: 2024-09-21T00:22:23.577+0000 7f04ac4ed700 -1 osd.4 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:22:24.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:23 smithi018 ceph-mon[78227]: osd.4 marked itself down and dead 2024-09-21T00:22:24.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:23 smithi018 ceph-mon[80555]: osd.4 marked itself down and dead 2024-09-21T00:22:24.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:24 smithi031 podman[73730]: 2024-09-21 00:22:24.090048374 +0000 UTC m=+0.683523664 container died fab2520e6b9f2046a1c8cb6abb4f58c04490edd18cbdfeaf140c165fd36e48ff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, version=8, ceph=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T00:22:24.763 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:24 smithi031 podman[73730]: 2024-09-21 00:22:24.569110485 +0000 UTC m=+1.162585776 container cleanup fab2520e6b9f2046a1c8cb6abb4f58c04490edd18cbdfeaf140c165fd36e48ff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, architecture=x86_64, vcs-type=git, ceph=True, distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-09-21T00:22:24.763 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:24 smithi031 bash[73730]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[78227]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[78227]: osdmap e139: 8 total, 7 up, 8 in 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[80555]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:24 smithi018 ceph-mon[80555]: osdmap e139: 8 total, 7 up, 8 in 2024-09-21T00:22:25.242 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:24 smithi031 ceph-mon[63863]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:25.242 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:24 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:25.242 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:24 smithi031 ceph-mon[63863]: osdmap e139: 8 total, 7 up, 8 in 2024-09-21T00:22:25.243 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:24 smithi031 podman[73747]: 2024-09-21 00:22:24.870587917 +0000 UTC m=+0.770116635 container remove fab2520e6b9f2046a1c8cb6abb4f58c04490edd18cbdfeaf140c165fd36e48ff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, io.openshift.tags=base centos centos-stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T00:22:25.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.144768257 +0000 UTC m=+0.016709476 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:22:25.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.24628654 +0000 UTC m=+0.118227747 container create af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T00:22:25.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.440466538 +0000 UTC m=+0.312407757 container init af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:22:25.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.443869442 +0000 UTC m=+0.315810649 container start af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-21T00:22:26.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:25 smithi031 ceph-mon[63863]: osdmap e140: 8 total, 7 up, 8 in 2024-09-21T00:22:26.002 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.525510754 +0000 UTC m=+0.397451959 container attach af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T00:22:26.002 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:25 smithi031 podman[73835]: 2024-09-21 00:22:25.64050048 +0000 UTC m=+0.512441694 container died af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, RELEASE=squid-9e2ac70, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:22:26.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:25 smithi018 ceph-mon[78227]: osdmap e140: 8 total, 7 up, 8 in 2024-09-21T00:22:26.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:25 smithi018 ceph-mon[80555]: osdmap e140: 8 total, 7 up, 8 in 2024-09-21T00:22:26.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 podman[73858]: 2024-09-21 00:22:26.295535067 +0000 UTC m=+0.649073848 container remove af780123dcebb34f418af8466bff311037e3c1b770f8839fb30c9dccac37e073 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:22:26.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T00:22:26.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 systemd[1]: Stopped Ceph osd.4 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:22:26.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.4.service: Consumed 18.461s CPU time. 2024-09-21T00:22:26.805 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 systemd[1]: Starting Ceph osd.4 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:22:26.805 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 podman[73959]: 2024-09-21 00:22:26.668086915 +0000 UTC m=+0.027803151 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:22:26.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:26 smithi031 ceph-mon[63863]: pgmap v184: 161 pgs: 2 activating+undersized, 9 peering, 29 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 98 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-21T00:22:26.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:26 smithi031 ceph-mon[63863]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T00:22:26.805 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:26 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:27.107 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:26 smithi031 podman[73959]: 2024-09-21 00:22:26.804704266 +0000 UTC m=+0.164420495 container create 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T00:22:27.108 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 podman[73959]: 2024-09-21 00:22:27.032294526 +0000 UTC m=+0.392010762 container init 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T00:22:27.108 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 podman[73959]: 2024-09-21 00:22:27.035638585 +0000 UTC m=+0.395354812 container start 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2) 2024-09-21T00:22:27.108 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 podman[73959]: 2024-09-21 00:22:27.107049449 +0000 UTC m=+0.466765678 container attach 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T00:22:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[78227]: pgmap v184: 161 pgs: 2 activating+undersized, 9 peering, 29 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 98 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-21T00:22:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[78227]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T00:22:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[80555]: pgmap v184: 161 pgs: 2 activating+undersized, 9 peering, 29 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 98 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-21T00:22:27.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[80555]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T00:22:27.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:26 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:27.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:22:27.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 bash[73959]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:22:27.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:22:27.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:27 smithi031 bash[73959]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:22:28.850 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-mon[63863]: pgmap v185: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:28.850 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T00:22:28.850 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T00:22:28.850 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T00:22:28.850 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T00:22:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:28 smithi018 ceph-mon[78227]: pgmap v185: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:28 smithi018 ceph-mon[80555]: pgmap v185: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:29.187 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T00:22:29.187 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T00:22:29.187 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T00:22:29.187 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate[73973]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 bash[73959]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T00:22:29.188 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:28 smithi031 podman[73959]: 2024-09-21 00:22:28.885026882 +0000 UTC m=+2.244743116 container died 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T00:22:29.898 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:29 smithi031 podman[74100]: 2024-09-21 00:22:29.643991049 +0000 UTC m=+0.747906905 container remove 18bd75971240ba957baeec84b4e6d3b558dc8a244a5688162048e88fa800153c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T00:22:29.898 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:29 smithi031 podman[74162]: 2024-09-21 00:22:29.770390379 +0000 UTC m=+0.017251290 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:22:29.899 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:29 smithi031 podman[74162]: 2024-09-21 00:22:29.898089824 +0000 UTC m=+0.144950712 container create 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T00:22:30.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:30 smithi031 podman[74162]: 2024-09-21 00:22:30.163960681 +0000 UTC m=+0.410821580 container init 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:22:30.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:30 smithi031 podman[74162]: 2024-09-21 00:22:30.167350198 +0000 UTC m=+0.414211093 container start 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T00:22:30.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:30 smithi031 ceph-osd[74181]: -- 172.21.15.31:0/1166632841 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560136d2ba40 con 0x560137af2400 2024-09-21T00:22:30.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:30 smithi031 bash[74162]: 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b 2024-09-21T00:22:30.501 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:30 smithi031 systemd[1]: Started Ceph osd.4 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:22:31.051 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:30 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:31.051 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:30 smithi031 ceph-mon[63863]: pgmap v186: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:31.051 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:31.051 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[78227]: pgmap v186: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[80555]: pgmap v186: 161 pgs: 2 activating+undersized, 9 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 105/723 objects degraded (14.523%) 2024-09-21T00:22:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:31.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:32.001 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:22:31.794+0000 7faf0c3b7640 -1 Falling back to public interface 2024-09-21T00:22:32.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:31 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:32.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:31 smithi031 ceph-mon[63863]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T00:22:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:31] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-09-21T00:22:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:31 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:31 smithi018 ceph-mon[78227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T00:22:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:31 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 124/723 objects degraded (17.151%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:32.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:31 smithi018 ceph-mon[80555]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T00:22:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:32 smithi018 ceph-mon[78227]: pgmap v187: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:32 smithi018 ceph-mon[80555]: pgmap v187: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:33.207 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:32 smithi031 ceph-mon[63863]: pgmap v187: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:35.576 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:22:35.339+0000 7faf0c3b7640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-21T00:22:35.577 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:22:35.368+0000 7faf0c3b7640 -1 osd.4 138 log_to_monitors true 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: pgmap v188: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.577 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: pgmap v188: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: pgmap v188: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:35.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:36.615 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:36 smithi031 ceph-mon[63863]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:36.615 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:36 smithi031 ceph-mon[63863]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:36.615 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:22:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:22:36.330+0000 7faf04162640 -1 osd.4 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:22:36.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:36 smithi018 ceph-mon[78227]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:36.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:36 smithi018 ceph-mon[78227]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:36.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:36 smithi018 ceph-mon[80555]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:36.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:36 smithi018 ceph-mon[80555]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T00:22:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:37 smithi031 ceph-mon[63863]: pgmap v189: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:37 smithi031 ceph-mon[63863]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T00:22:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:37 smithi031 ceph-mon[63863]: osdmap e141: 8 total, 7 up, 8 in 2024-09-21T00:22:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:37 smithi031 ceph-mon[63863]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:37 smithi031 ceph-mon[63863]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[78227]: pgmap v189: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[78227]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[78227]: osdmap e141: 8 total, 7 up, 8 in 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[78227]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[78227]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[80555]: pgmap v189: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 180 MiB used, 715 GiB / 715 GiB avail; 124/723 objects degraded (17.151%) 2024-09-21T00:22:37.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[80555]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T00:22:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[80555]: osdmap e141: 8 total, 7 up, 8 in 2024-09-21T00:22:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[80555]: from='osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:37.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:37 smithi018 ceph-mon[80555]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:22:38.125+0000 7fd0517a1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555] boot 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: pgmap v192: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: pgmap v193: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: pgmap v194: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[78227]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555] boot 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: pgmap v192: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: pgmap v193: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: pgmap v194: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T00:22:38.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:38 smithi018 ceph-mon[80555]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T00:22:38.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:22:38.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: osd.4 [v2:172.21.15.31:6800/2568178555,v1:172.21.15.31:6801/2568178555] boot 2024-09-21T00:22:38.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T00:22:38.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T00:22:38.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: pgmap v192: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: pgmap v193: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: pgmap v194: 161 pgs: 41 active+undersized, 26 active+undersized+degraded, 94 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 124/723 objects degraded (17.151%) 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:38.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T00:22:38.753 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:38 smithi031 ceph-mon[63863]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T00:22:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:40 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:40 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:40 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T00:22:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:41 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:41 smithi018 ceph-mon[78227]: pgmap v196: 161 pgs: 35 active+undersized, 24 active+undersized+degraded, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 112/723 objects degraded (15.491%) 2024-09-21T00:22:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:41 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:41 smithi018 ceph-mon[80555]: pgmap v196: 161 pgs: 35 active+undersized, 24 active+undersized+degraded, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 112/723 objects degraded (15.491%) 2024-09-21T00:22:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:41 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:41 smithi031 ceph-mon[63863]: pgmap v196: 161 pgs: 35 active+undersized, 24 active+undersized+degraded, 102 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 112/723 objects degraded (15.491%) 2024-09-21T00:22:42.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:42 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded) 2024-09-21T00:22:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:42 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded) 2024-09-21T00:22:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:41] "GET /metrics HTTP/1.1" 200 38207 "" "Prometheus/2.43.0" 2024-09-21T00:22:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:42 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 112/723 objects degraded (15.491%), 24 pgs degraded) 2024-09-21T00:22:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:43 smithi018 ceph-mon[78227]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:22:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:43 smithi018 ceph-mon[80555]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:22:43.477 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:43 smithi031 ceph-mon[63863]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:22:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:45 smithi018 ceph-mon[78227]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-09-21T00:22:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:45 smithi018 ceph-mon[80555]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-09-21T00:22:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:45 smithi031 ceph-mon[63863]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-09-21T00:22:47.166 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:22:47.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:47 smithi018 ceph-mon[78227]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:47 smithi018 ceph-mon[80555]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:47 smithi031 ceph-mon[63863]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:47.850 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (19m) 48s ago 27m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (6m) 13s ago 27m 78.9M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 48s ago 27m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (6m) 13s ago 32m 441M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (16m) 48s ago 34m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (4m) 48s ago 34m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:22:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (5m) 13s ago 32m 45.6M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (4m) 48s ago 32m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (20m) 48s ago 28m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (19m) 13s ago 28m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (3m) 48s ago 32m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (2m) 48s ago 31m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (107s) 48s ago 31m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (53s) 48s ago 31m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (18s) 13s ago 30m 12.1M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:22:47.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (30m) 13s ago 30m 57.0M 3249M 17.2.0 e1d6a67b021e b6cc23ae9924 2024-09-21T00:22:47.853 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (29m) 13s ago 29m 57.1M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:22:47.853 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (29m) 13s ago 29m 61.1M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:22:47.853 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (7m) 13s ago 28m 69.2M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:22:47.853 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (27m) 48s ago 27m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:22:47.853 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (27m) 13s ago 27m 81.7M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:22:48.268 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:22:48.268 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:22:48.268 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 5 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:22:48.269 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T00:22:48.270 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 10 2024-09-21T00:22:48.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:22:48.270 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:22:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:48 smithi018 ceph-mon[78227]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:48.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:48 smithi018 ceph-mon[80555]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:48 smithi031 ceph-mon[63863]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:48.623 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:22:48.623 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:22:48.623 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:22:48.623 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-21T00:22:48.624 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:22:48.624 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "5/8 daemons upgraded", 2024-09-21T00:22:48.624 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:22:48.624 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:22:48.624 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:22:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[78227]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[78227]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[78227]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:22:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/623639359' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[80555]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[80555]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[80555]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:22:49.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:49 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/623639359' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:49 smithi031 ceph-mon[63863]: from='client.44481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:49 smithi031 ceph-mon[63863]: from='client.44487 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:49 smithi031 ceph-mon[63863]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:22:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:49 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/623639359' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:22:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[80555]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:50.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[78227]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:50.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:50.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:50.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:50 smithi031 ceph-mon[63863]: from='client.44499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:22:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:22:50.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:51 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:51 smithi018 ceph-mon[78227]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:22:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:51 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:51.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:51 smithi018 ceph-mon[80555]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:22:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:51 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:22:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:51 smithi031 ceph-mon[63863]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-21T00:22:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:22:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:22:51] "GET /metrics HTTP/1.1" 200 38206 "" "Prometheus/2.43.0" 2024-09-21T00:22:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:53 smithi018 ceph-mon[78227]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:53.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:53 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:53 smithi018 ceph-mon[80555]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:53.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:53 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:53 smithi031 ceph-mon[63863]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:53 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:54.435 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:54 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:54.436 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:54 smithi031 ceph-mon[63863]: Upgrade: osd.5 is safe to restart 2024-09-21T00:22:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:54 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:54 smithi018 ceph-mon[78227]: Upgrade: osd.5 is safe to restart 2024-09-21T00:22:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:54 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T00:22:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:54 smithi018 ceph-mon[80555]: Upgrade: osd.5 is safe to restart 2024-09-21T00:22:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:55 smithi031 ceph-mon[63863]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:22:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[78227]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[80555]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:22:55.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T00:22:55.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:22:56.428 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:56 smithi031 ceph-mon[63863]: Upgrade: Updating osd.5 2024-09-21T00:22:56.428 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:56 smithi031 ceph-mon[63863]: Deploying daemon osd.5 on smithi031 2024-09-21T00:22:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:56 smithi018 ceph-mon[78227]: Upgrade: Updating osd.5 2024-09-21T00:22:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:56 smithi018 ceph-mon[78227]: Deploying daemon osd.5 on smithi031 2024-09-21T00:22:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:56 smithi018 ceph-mon[80555]: Upgrade: Updating osd.5 2024-09-21T00:22:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:56 smithi018 ceph-mon[80555]: Deploying daemon osd.5 on smithi031 2024-09-21T00:22:57.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:57 smithi031 ceph-mon[63863]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:57.616 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:57 smithi031 systemd[1]: Stopping Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:22:57.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:57 smithi018 ceph-mon[78227]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:57.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:57 smithi018 ceph-mon[80555]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:22:58.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[30750]: 2024-09-21T00:22:57.614+0000 7f912aadf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:22:58.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[30750]: 2024-09-21T00:22:57.614+0000 7f912aadf700 -1 osd.5 143 *** Got signal Terminated *** 2024-09-21T00:22:58.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[30750]: 2024-09-21T00:22:57.614+0000 7f912aadf700 -1 osd.5 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:22:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:58 smithi031 ceph-mon[63863]: osd.5 marked itself down and dead 2024-09-21T00:22:58.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:58 smithi018 ceph-mon[80555]: osd.5 marked itself down and dead 2024-09-21T00:22:58.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:58 smithi018 ceph-mon[78227]: osd.5 marked itself down and dead 2024-09-21T00:22:59.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:58 smithi031 podman[78343]: 2024-09-21 00:22:58.725384589 +0000 UTC m=+1.229132000 container died b6cc23ae9924ebe14b9a40a727a59ddaed90f6eb2fc01e2dfa0fc41fb4c167ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T00:22:59.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:59 smithi031 ceph-mon[63863]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:59.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:59 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:59.381 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:22:59 smithi031 ceph-mon[63863]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T00:22:59.381 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78343]: 2024-09-21 00:22:59.095827179 +0000 UTC m=+1.599574581 container cleanup b6cc23ae9924ebe14b9a40a727a59ddaed90f6eb2fc01e2dfa0fc41fb4c167ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, release=754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T00:22:59.381 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 bash[78343]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[80555]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[80555]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[78227]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:22:59.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:22:59 smithi018 ceph-mon[78227]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T00:22:59.737 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78370]: 2024-09-21 00:22:59.380588257 +0000 UTC m=+0.648403648 container remove b6cc23ae9924ebe14b9a40a727a59ddaed90f6eb2fc01e2dfa0fc41fb4c167ad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T00:23:00.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78461]: 2024-09-21 00:22:59.639601456 +0000 UTC m=+0.016592103 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:00.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78461]: 2024-09-21 00:22:59.748033656 +0000 UTC m=+0.125024288 container create 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T00:23:00.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78461]: 2024-09-21 00:22:59.925222988 +0000 UTC m=+0.302213627 container init 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T00:23:00.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:22:59 smithi031 podman[78461]: 2024-09-21 00:22:59.928738024 +0000 UTC m=+0.305728659 container start 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T00:23:00.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 podman[78461]: 2024-09-21 00:23:00.000201175 +0000 UTC m=+0.377191824 container attach 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:23:00.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:00 smithi031 ceph-mon[63863]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T00:23:00.251 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 podman[78461]: 2024-09-21 00:23:00.123776557 +0000 UTC m=+0.500767206 container died 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T00:23:00.660 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 podman[78483]: 2024-09-21 00:23:00.629083575 +0000 UTC m=+0.499296740 container remove 6dfb0d5bbd1a1b349bcb4aabf6786d1869565cc08fce4e0ddd2a8e0d68c97546 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T00:23:00.660 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T00:23:00.660 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 systemd[1]: Stopped Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:23:00.660 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service: Consumed 17.391s CPU time. 2024-09-21T00:23:00.674 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:00 smithi018 ceph-mon[78227]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T00:23:00.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:00 smithi018 ceph-mon[80555]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T00:23:01.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:00 smithi031 systemd[1]: Starting Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:23:01.374 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:01 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:01.374 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:01 smithi031 ceph-mon[63863]: pgmap v208: 161 pgs: 12 active+undersized, 17 stale+active+clean, 1 active+undersized+degraded, 131 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 11/723 objects degraded (1.521%) 2024-09-21T00:23:01.374 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:01 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 11/723 objects degraded (1.521%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:23:01.375 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 podman[78585]: 2024-09-21 00:23:00.968512269 +0000 UTC m=+0.016904662 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:01.375 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 podman[78585]: 2024-09-21 00:23:01.088371437 +0000 UTC m=+0.136763865 container create 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T00:23:01.375 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 podman[78585]: 2024-09-21 00:23:01.299155362 +0000 UTC m=+0.347547741 container init 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T00:23:01.375 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 podman[78585]: 2024-09-21 00:23:01.303221187 +0000 UTC m=+0.351613568 container start 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T00:23:01.375 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 podman[78585]: 2024-09-21 00:23:01.373785807 +0000 UTC m=+0.422178193 container attach 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[80555]: pgmap v208: 161 pgs: 12 active+undersized, 17 stale+active+clean, 1 active+undersized+degraded, 131 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 11/723 objects degraded (1.521%) 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 11/723 objects degraded (1.521%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[78227]: pgmap v208: 161 pgs: 12 active+undersized, 17 stale+active+clean, 1 active+undersized+degraded, 131 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 11/723 objects degraded (1.521%) 2024-09-21T00:23:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 11/723 objects degraded (1.521%), 1 pg degraded (PG_DEGRADED) 2024-09-21T00:23:01.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:01.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 bash[78585]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:01.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:01.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:01 smithi031 bash[78585]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:01] "GET /metrics HTTP/1.1" 200 38206 "" "Prometheus/2.43.0" 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T00:23:02.802 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T00:23:03.145 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T00:23:03.146 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate[78600]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T00:23:03.146 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 bash[78585]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T00:23:03.146 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:02 smithi031 podman[78585]: 2024-09-21 00:23:02.833870336 +0000 UTC m=+1.882262729 container died 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:23:03.428 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:03 smithi031 ceph-mon[63863]: pgmap v209: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:03 smithi018 ceph-mon[78227]: pgmap v209: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:03.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:03 smithi018 ceph-mon[80555]: pgmap v209: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:03.749 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:03 smithi031 podman[78737]: 2024-09-21 00:23:03.427552343 +0000 UTC m=+0.586974576 container remove 83bc6ce4c124d4f7310b6dfe49bcf3d033afef2cfd0363e1f2f163b5cdf0232a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-activate, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T00:23:03.749 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:03 smithi031 podman[78793]: 2024-09-21 00:23:03.554427174 +0000 UTC m=+0.016820523 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:03.749 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:03 smithi031 podman[78793]: 2024-09-21 00:23:03.748783129 +0000 UTC m=+0.211176480 container create add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:23:04.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:04 smithi031 podman[78793]: 2024-09-21 00:23:04.122787032 +0000 UTC m=+0.585180392 container init add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T00:23:04.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:04 smithi031 podman[78793]: 2024-09-21 00:23:04.126674237 +0000 UTC m=+0.589067586 container start add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70) 2024-09-21T00:23:04.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:04 smithi031 ceph-osd[78811]: -- 172.21.15.31:0/3604865459 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e400095a40 con 0x55e400e5c000 2024-09-21T00:23:04.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:04 smithi031 bash[78793]: add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 2024-09-21T00:23:04.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:04 smithi031 systemd[1]: Started Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[80555]: pgmap v210: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[78227]: pgmap v210: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:05.749 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-mon[63863]: pgmap v210: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:05.749 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.749 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.749 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:05.749 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:06.001 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:05 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:23:05.747+0000 7fb6c7529640 -1 Falling back to public interface 2024-09-21T00:23:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:07 smithi031 ceph-mon[63863]: pgmap v211: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:07 smithi018 ceph-mon[78227]: pgmap v211: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:07.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:07 smithi018 ceph-mon[80555]: pgmap v211: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:08 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 79/723 objects degraded (10.927%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:08 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 79/723 objects degraded (10.927%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:08.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:08 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 79/723 objects degraded (10.927%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:09.275 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:23:09.010+0000 7fb6c7529640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-21T00:23:09.275 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:23:09.037+0000 7fb6c7529640 -1 osd.5 143 log_to_monitors true 2024-09-21T00:23:09.631 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:23:09.455+0000 7fb6bf2d4640 -1 osd.5 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: pgmap v212: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.632 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:09 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: pgmap v212: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: pgmap v212: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:09.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:09 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:10 smithi031 ceph-mon[63863]: pgmap v214: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[80555]: pgmap v214: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:10.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T00:23:10.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T00:23:10.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: from='osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:10.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:10 smithi018 ceph-mon[78227]: pgmap v214: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 79/723 objects degraded (10.927%) 2024-09-21T00:23:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:11 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:11 smithi031 ceph-mon[63863]: osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097] boot 2024-09-21T00:23:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:11 smithi031 ceph-mon[63863]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T00:23:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:11 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:23:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:11 smithi031 ceph-mon[63863]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[78227]: osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097] boot 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[78227]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[78227]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T00:23:11.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:11.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[80555]: osd.5 [v2:172.21.15.31:6808/4042377097,v1:172.21.15.31:6809/4042377097] boot 2024-09-21T00:23:11.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[80555]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T00:23:11.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T00:23:11.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-mon[80555]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T00:23:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:11] "GET /metrics HTTP/1.1" 200 38216 "" "Prometheus/2.43.0" 2024-09-21T00:23:12.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:12 smithi031 ceph-mon[63863]: pgmap v217: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:12.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:12 smithi018 ceph-mon[78227]: pgmap v217: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:12.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:12 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:23:12.609+0000 7fd0517a1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:12.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:12 smithi018 ceph-mon[80555]: pgmap v217: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: pgmap v218: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: pgmap v219: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:13.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:13 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: pgmap v218: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: pgmap v219: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:13.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: pgmap v218: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: pgmap v219: 161 pgs: 10 peering, 30 active+undersized, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 70/723 objects degraded (9.682%) 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:13.927 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:13 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 70/723 objects degraded (9.682%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:15.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:15 smithi018 ceph-mon[78227]: pgmap v220: 161 pgs: 10 peering, 16 active+undersized, 7 active+undersized+degraded, 128 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-09-21T00:23:15.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:15 smithi018 ceph-mon[80555]: pgmap v220: 161 pgs: 10 peering, 16 active+undersized, 7 active+undersized+degraded, 128 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-09-21T00:23:16.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:15 smithi031 ceph-mon[63863]: pgmap v220: 161 pgs: 10 peering, 16 active+undersized, 7 active+undersized+degraded, 128 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s; 33/723 objects degraded (4.564%) 2024-09-21T00:23:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:16 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 7 pgs degraded) 2024-09-21T00:23:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:16 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 7 pgs degraded) 2024-09-21T00:23:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:16 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/723 objects degraded (4.564%), 7 pgs degraded) 2024-09-21T00:23:17.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:17 smithi018 ceph-mon[78227]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:23:17.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:17 smithi018 ceph-mon[80555]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:23:18.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:17 smithi031 ceph-mon[63863]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:23:19.004 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (20m) 80s ago 28m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (7m) 11s ago 28m 79.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 80s ago 27m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (7m) 11s ago 33m 441M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (17m) 80s ago 34m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (5m) 80s ago 34m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (6m) 11s ago 33m 47.1M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:23:19.709 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (5m) 80s ago 33m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (20m) 80s ago 29m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (20m) 11s ago 29m 16.5M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (3m) 80s ago 32m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (3m) 80s ago 32m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (2m) 80s ago 32m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (85s) 80s ago 31m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (49s) 11s ago 31m 45.4M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (16s) 11s ago 30m 11.6M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (30m) 11s ago 30m 58.8M 3249M 17.2.0 e1d6a67b021e 187cafff1016 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (30m) 11s ago 30m 61.6M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (7m) 11s ago 29m 69.2M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (27m) 80s ago 27m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:23:19.710 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (27m) 11s ago 27m 81.9M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[78227]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 725 B/s rd, 0 op/s 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[78227]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[80555]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 725 B/s rd, 0 op/s 2024-09-21T00:23:19.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[80555]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:19.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:19 smithi031 ceph-mon[63863]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 725 B/s rd, 0 op/s 2024-09-21T00:23:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:19 smithi031 ceph-mon[63863]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:20.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:20.135 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:23:20.135 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:23:20.135 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:23:20.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:20.135 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 6 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:23:20.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:20.137 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:23:20.137 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T00:23:20.137 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 11 2024-09-21T00:23:20.137 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:23:20.137 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:23:20.524 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:23:20.524 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:23:20.524 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:23:20.524 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-21T00:23:20.524 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:23:20.525 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "6/8 daemons upgraded", 2024-09-21T00:23:20.525 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:23:20.525 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:23:20.525 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[78227]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[78227]: from='client.44523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1380109864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[80555]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:20.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[80555]: from='client.44523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:20.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:20 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1380109864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:20 smithi031 ceph-mon[63863]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:20 smithi031 ceph-mon[63863]: from='client.44523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:20 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1380109864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:21.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:21 smithi018 ceph-mon[80555]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-21T00:23:21.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:21 smithi018 ceph-mon[80555]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:21.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:21 smithi018 ceph-mon[78227]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-21T00:23:21.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:21 smithi018 ceph-mon[78227]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:22.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:21 smithi031 ceph-mon[63863]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s 2024-09-21T00:23:22.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:21 smithi031 ceph-mon[63863]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:21] "GET /metrics HTTP/1.1" 200 38221 "" "Prometheus/2.43.0" 2024-09-21T00:23:23.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:23 smithi018 ceph-mon[78227]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:23:23.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:23 smithi018 ceph-mon[80555]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:23:24.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:23 smithi031 ceph-mon[63863]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:23:25.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:25 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:25.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:25 smithi031 ceph-mon[63863]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:25 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:25.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:25 smithi018 ceph-mon[78227]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:25.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:25 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:25.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:25 smithi018 ceph-mon[80555]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:27.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:27 smithi018 ceph-mon[78227]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:27.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:27 smithi018 ceph-mon[80555]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:28.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:27 smithi031 ceph-mon[63863]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:28 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:28 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:28 smithi031 ceph-mon[63863]: Upgrade: osd.6 is safe to restart 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[78227]: Upgrade: osd.6 is safe to restart 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T00:23:28.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:28 smithi018 ceph-mon[80555]: Upgrade: osd.6 is safe to restart 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: Upgrade: Updating osd.6 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:30.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[78227]: Deploying daemon osd.6 on smithi031 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: Upgrade: Updating osd.6 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:30.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:30 smithi018 ceph-mon[80555]: Deploying daemon osd.6 on smithi031 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: Upgrade: Updating osd.6 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:30.500 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:30 smithi031 ceph-mon[63863]: Deploying daemon osd.6 on smithi031 2024-09-21T00:23:31.305 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:31 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:31 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:31 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:32 smithi018 ceph-mon[78227]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:32 smithi018 ceph-mon[80555]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:31] "GET /metrics HTTP/1.1" 200 38221 "" "Prometheus/2.43.0" 2024-09-21T00:23:32.448 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:32 smithi031 ceph-mon[63863]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:23:32.448 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:32 smithi031 systemd[1]: Stopping Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:23:32.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[33628]: 2024-09-21T00:23:32.445+0000 7f9003032700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:23:32.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[33628]: 2024-09-21T00:23:32.445+0000 7f9003032700 -1 osd.6 148 *** Got signal Terminated *** 2024-09-21T00:23:32.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[33628]: 2024-09-21T00:23:32.445+0000 7f9003032700 -1 osd.6 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:23:33.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:33 smithi018 ceph-mon[78227]: osd.6 marked itself down and dead 2024-09-21T00:23:33.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:33 smithi018 ceph-mon[80555]: osd.6 marked itself down and dead 2024-09-21T00:23:33.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:33 smithi031 ceph-mon[63863]: osd.6 marked itself down and dead 2024-09-21T00:23:33.915 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:33 smithi031 podman[83286]: 2024-09-21 00:23:33.676462218 +0000 UTC m=+1.336893967 container died 187cafff101641d7ec4b17b4400ee98ff3f698baa512aae020d490b81bfdb468 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T00:23:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:34 smithi031 ceph-mon[63863]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:34 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:23:34.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:34 smithi031 ceph-mon[63863]: osdmap e149: 8 total, 7 up, 8 in 2024-09-21T00:23:34.252 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:34 smithi031 podman[83286]: 2024-09-21 00:23:34.029802723 +0000 UTC m=+1.690234466 container cleanup 187cafff101641d7ec4b17b4400ee98ff3f698baa512aae020d490b81bfdb468 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., name=centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True) 2024-09-21T00:23:34.252 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:34 smithi031 bash[83286]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[78227]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[78227]: osdmap e149: 8 total, 7 up, 8 in 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[80555]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:23:34.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:34 smithi018 ceph-mon[80555]: osdmap e149: 8 total, 7 up, 8 in 2024-09-21T00:23:34.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:34 smithi031 podman[83301]: 2024-09-21 00:23:34.381519801 +0000 UTC m=+0.700491195 container remove 187cafff101641d7ec4b17b4400ee98ff3f698baa512aae020d490b81bfdb468 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_CLEAN=True) 2024-09-21T00:23:35.102 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:34 smithi031 podman[83391]: 2024-09-21 00:23:34.798930184 +0000 UTC m=+0.106103851 container create 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T00:23:35.102 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:34 smithi031 podman[83391]: 2024-09-21 00:23:34.70950226 +0000 UTC m=+0.016675928 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:35.102 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 podman[83391]: 2024-09-21 00:23:35.018724452 +0000 UTC m=+0.325898129 container init 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T00:23:35.103 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 podman[83391]: 2024-09-21 00:23:35.022652499 +0000 UTC m=+0.329826167 container start 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70) 2024-09-21T00:23:35.103 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 podman[83391]: 2024-09-21 00:23:35.101713584 +0000 UTC m=+0.408887263 container attach 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[78227]: osdmap e150: 8 total, 7 up, 8 in 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[80555]: osdmap e150: 8 total, 7 up, 8 in 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:35.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:35 smithi031 ceph-mon[63863]: osdmap e150: 8 total, 7 up, 8 in 2024-09-21T00:23:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:35.501 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 podman[83391]: 2024-09-21 00:23:35.220826703 +0000 UTC m=+0.528000381 container died 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:23:36.135 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 podman[83414]: 2024-09-21 00:23:35.805516297 +0000 UTC m=+0.578823888 container remove 6954b7046bcd2849ff5c7af0f9f86d0ac9aa848d987e87c13c13e6d08261a01b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T00:23:36.135 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T00:23:36.136 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 systemd[1]: Stopped Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:23:36.136 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:35 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service: Consumed 16.532s CPU time. 2024-09-21T00:23:36.136 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 systemd[1]: Starting Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:23:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:36 smithi018 ceph-mon[78227]: pgmap v232: 161 pgs: 6 active+undersized, 11 stale+active+clean, 4 active+undersized+degraded, 140 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 13/723 objects degraded (1.798%) 2024-09-21T00:23:36.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:36 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:36 smithi018 ceph-mon[80555]: pgmap v232: 161 pgs: 6 active+undersized, 11 stale+active+clean, 4 active+undersized+degraded, 140 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 13/723 objects degraded (1.798%) 2024-09-21T00:23:36.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:36 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:36.450 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:36 smithi031 ceph-mon[63863]: pgmap v232: 161 pgs: 6 active+undersized, 11 stale+active+clean, 4 active+undersized+degraded, 140 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 13/723 objects degraded (1.798%) 2024-09-21T00:23:36.450 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:36 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 13/723 objects degraded (1.798%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:36.450 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 podman[83511]: 2024-09-21 00:23:36.340053105 +0000 UTC m=+0.103562867 container create 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70) 2024-09-21T00:23:36.450 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 podman[83511]: 2024-09-21 00:23:36.253459981 +0000 UTC m=+0.016969740 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 podman[83511]: 2024-09-21 00:23:36.559420132 +0000 UTC m=+0.322929894 container init 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 podman[83511]: 2024-09-21 00:23:36.562993796 +0000 UTC m=+0.326503558 container start 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 podman[83511]: 2024-09-21 00:23:36.634321444 +0000 UTC m=+0.397831204 container attach 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 bash[83511]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:36.712 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:36 smithi031 bash[83511]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:23:37.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T00:23:37.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T00:23:37.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T00:23:37.752 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T00:23:38.106 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T00:23:38.107 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T00:23:38.107 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T00:23:38.107 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate[83525]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T00:23:38.107 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 bash[83511]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T00:23:38.107 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:37 smithi031 podman[83511]: 2024-09-21 00:23:37.808734645 +0000 UTC m=+1.572244412 container died 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:23:38.414 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:38 smithi031 ceph-mon[63863]: pgmap v233: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:38.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:38 smithi018 ceph-mon[80555]: pgmap v233: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:38.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:38 smithi018 ceph-mon[78227]: pgmap v233: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:38.737 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 podman[83660]: 2024-09-21 00:23:38.41362234 +0000 UTC m=+0.597740705 container remove 598b7f2f94a4df3fe04fa743170cc43dc9d4e3cc9eaf7ab5071cc61673e55025 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T00:23:38.737 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 podman[83718]: 2024-09-21 00:23:38.631132737 +0000 UTC m=+0.112155152 container create 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:23:38.737 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 podman[83718]: 2024-09-21 00:23:38.536492662 +0000 UTC m=+0.017515091 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:23:39.251 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 podman[83718]: 2024-09-21 00:23:38.84979806 +0000 UTC m=+0.330820476 container init 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T00:23:39.251 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 podman[83718]: 2024-09-21 00:23:38.85319186 +0000 UTC m=+0.334214278 container start 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, GIT_CLEAN=True, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T00:23:39.251 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 ceph-osd[83737]: -- 172.21.15.31:0/2916779249 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55557cb73a40 con 0x55557cb51c00 2024-09-21T00:23:39.251 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 bash[83718]: 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 2024-09-21T00:23:39.252 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:38 smithi031 systemd[1]: Started Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:23:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:39 smithi031 ceph-mon[63863]: pgmap v234: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[78227]: pgmap v234: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[80555]: pgmap v234: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:40.749 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:23:40.747+0000 7f90540f3640 -1 Falling back to public interface 2024-09-21T00:23:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:41 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:40 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:40 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:42.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:41 smithi018 ceph-mon[78227]: pgmap v235: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:42.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:41 smithi018 ceph-mon[80555]: pgmap v235: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:41] "GET /metrics HTTP/1.1" 200 38223 "" "Prometheus/2.43.0" 2024-09-21T00:23:42.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:41 smithi031 ceph-mon[63863]: pgmap v235: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:43 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:43.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:43 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:43 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:44.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-mon[63863]: pgmap v236: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:44.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.273 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.274 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[78227]: pgmap v236: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[80555]: pgmap v236: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:44.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:23:44.271+0000 7f90540f3640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-21T00:23:44.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:23:44.303+0000 7f90540f3640 -1 osd.6 148 log_to_monitors true 2024-09-21T00:23:45.193 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:45 smithi031 ceph-mon[63863]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:45.193 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:45 smithi031 ceph-mon[63863]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:45.194 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:23:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:23:45.036+0000 7f904be9e640 -1 osd.6 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:23:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:45 smithi018 ceph-mon[78227]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:45.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:45 smithi018 ceph-mon[78227]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:45 smithi018 ceph-mon[80555]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:45.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:45 smithi018 ceph-mon[80555]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T00:23:46.280 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:46 smithi031 ceph-mon[63863]: pgmap v237: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:46.280 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:46 smithi031 ceph-mon[63863]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T00:23:46.280 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:46 smithi031 ceph-mon[63863]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:46.280 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:46 smithi031 ceph-mon[63863]: osdmap e151: 8 total, 7 up, 8 in 2024-09-21T00:23:46.281 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:46 smithi031 ceph-mon[63863]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[78227]: pgmap v237: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[78227]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[78227]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[78227]: osdmap e151: 8 total, 7 up, 8 in 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[78227]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:46.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[80555]: pgmap v237: 161 pgs: 33 active+undersized, 20 active+undersized+degraded, 108 active+clean; 585 KiB data, 212 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-09-21T00:23:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[80555]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T00:23:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[80555]: from='osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[80555]: osdmap e151: 8 total, 7 up, 8 in 2024-09-21T00:23:46.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:46 smithi018 ceph-mon[80555]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:23:47.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:23:47.267+0000 7fd0517a1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[78227]: osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648] boot 2024-09-21T00:23:47.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[78227]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T00:23:47.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:23:47.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[78227]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T00:23:47.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:47.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[80555]: osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648] boot 2024-09-21T00:23:47.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[80555]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T00:23:47.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:23:47.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:47 smithi018 ceph-mon[80555]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T00:23:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:47 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:23:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:47 smithi031 ceph-mon[63863]: osd.6 [v2:172.21.15.31:6816/2555528648,v1:172.21.15.31:6817/2555528648] boot 2024-09-21T00:23:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:47 smithi031 ceph-mon[63863]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T00:23:47.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:47 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T00:23:47.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:47 smithi031 ceph-mon[63863]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T00:23:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: pgmap v240: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: pgmap v242: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: pgmap v243: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:48.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: pgmap v240: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: pgmap v242: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: pgmap v243: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.427 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:48.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:48 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: pgmap v240: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:23:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: pgmap v242: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: pgmap v243: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-21T00:23:48.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:48 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T00:23:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:50 smithi031 ceph-mon[63863]: pgmap v244: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:50.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[78227]: pgmap v244: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:50.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:50.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:50.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[80555]: pgmap v244: 161 pgs: 8 peering, 28 active+undersized, 17 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 71/723 objects degraded (9.820%) 2024-09-21T00:23:50.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:23:50.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:23:50.903 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:23:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:51 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:51.589 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (21m) 112s ago 29m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (7m) 8s ago 28m 80.0M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 112s ago 28m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (8m) 8s ago 33m 441M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (17m) 112s ago 35m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (5m) 112s ago 35m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (6m) 8s ago 33m 47.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (5m) 112s ago 34m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (21m) 112s ago 29m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (20m) 8s ago 29m 16.2M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 112s ago 33m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:23:51.590 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (3m) 112s ago 32m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (2m) 112s ago 32m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (117s) 112s ago 32m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (81s) 8s ago 31m 45.9M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (48s) 8s ago 31m 44.0M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (13s) 8s ago 30m 12.1M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (30m) 8s ago 30m 61.8M 3249M 17.2.0 e1d6a67b021e c0a826304bb3 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (8m) 8s ago 29m 69.2M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (28m) 112s ago 28m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:23:51.591 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (28m) 8s ago 28m 82.0M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:23:51.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:51 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:51.591 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:51 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:23:52.006 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:23:52.006 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:23:52.006 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:23:52.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:52.006 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 7 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 12 2024-09-21T00:23:52.007 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:23:52.008 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:23:52.174 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:23:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:23:51] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "7/8 daemons upgraded", 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T00:23:52.358 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:23:52.359 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:23:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:52 smithi031 ceph-mon[63863]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:52 smithi031 ceph-mon[63863]: pgmap v245: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 599 B/s rd, 0 op/s 2024-09-21T00:23:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:52 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded) 2024-09-21T00:23:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:52 smithi031 ceph-mon[63863]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:52 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2279759939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[78227]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[78227]: pgmap v245: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 599 B/s rd, 0 op/s 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded) 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[78227]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2279759939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[80555]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[80555]: pgmap v245: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 599 B/s rd, 0 op/s 2024-09-21T00:23:52.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 17 pgs degraded) 2024-09-21T00:23:52.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[80555]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:52.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:52 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2279759939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:23:53.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:53 smithi031 ceph-mon[63863]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:53.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:53 smithi018 ceph-mon[78227]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:53.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:53 smithi018 ceph-mon[80555]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:54.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:54 smithi031 ceph-mon[63863]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:54.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:54 smithi031 ceph-mon[63863]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 461 B/s rd, 0 op/s 2024-09-21T00:23:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:54 smithi018 ceph-mon[80555]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:54 smithi018 ceph-mon[80555]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 461 B/s rd, 0 op/s 2024-09-21T00:23:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:54 smithi018 ceph-mon[78227]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:23:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:54 smithi018 ceph-mon[78227]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 461 B/s rd, 0 op/s 2024-09-21T00:23:56.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:56 smithi031 ceph-mon[63863]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 378 B/s rd, 0 op/s 2024-09-21T00:23:56.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:56 smithi018 ceph-mon[80555]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 378 B/s rd, 0 op/s 2024-09-21T00:23:56.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:56 smithi018 ceph-mon[78227]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 378 B/s rd, 0 op/s 2024-09-21T00:23:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:23:58 smithi031 ceph-mon[63863]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-09-21T00:23:58.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:23:58 smithi018 ceph-mon[78227]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-09-21T00:23:58.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:23:58 smithi018 ceph-mon[80555]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-09-21T00:24:00.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:00 smithi031 ceph-mon[63863]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-21T00:24:00.561 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:00 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:00.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:00 smithi018 ceph-mon[78227]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-21T00:24:00.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:00 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:00.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:00 smithi018 ceph-mon[80555]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-09-21T00:24:00.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:00 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:01 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:01.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:01 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:01.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:01 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:01] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-09-21T00:24:02.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:02 smithi031 ceph-mon[63863]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:24:02.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:02 smithi018 ceph-mon[78227]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:24:02.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:02 smithi018 ceph-mon[80555]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:24:03.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:03 smithi031 ceph-mon[63863]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:03 smithi031 ceph-mon[63863]: Upgrade: osd.7 is safe to restart 2024-09-21T00:24:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[78227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[78227]: Upgrade: osd.7 is safe to restart 2024-09-21T00:24:03.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[80555]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T00:24:03.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:03 smithi018 ceph-mon[80555]: Upgrade: osd.7 is safe to restart 2024-09-21T00:24:04.445 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:04 smithi031 ceph-mon[63863]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:04.445 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:04.445 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:24:04.445 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[78227]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[80555]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T00:24:04.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:05.410 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:05 smithi031 ceph-mon[63863]: Upgrade: Updating osd.7 2024-09-21T00:24:05.410 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:05 smithi031 ceph-mon[63863]: Deploying daemon osd.7 on smithi031 2024-09-21T00:24:05.410 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:05.410 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[80555]: Upgrade: Updating osd.7 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[80555]: Deploying daemon osd.7 on smithi031 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[78227]: Upgrade: Updating osd.7 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[78227]: Deploying daemon osd.7 on smithi031 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:05.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]: dispatch 2024-09-21T00:24:06.440 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]: dispatch 2024-09-21T00:24:06.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]: dispatch 2024-09-21T00:24:06.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:06 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]: dispatch 2024-09-21T00:24:06.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:06 smithi031 systemd[1]: Stopping Ceph osd.7 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:24:07.244 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[36535]: 2024-09-21T00:24:06.910+0000 7f92a4c54700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:24:07.244 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[36535]: 2024-09-21T00:24:06.910+0000 7f92a4c54700 -1 osd.7 154 *** Got signal Terminated *** 2024-09-21T00:24:07.244 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:06 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[36535]: 2024-09-21T00:24:06.910+0000 7f92a4c54700 -1 osd.7 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:24:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T00:24:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-21T00:24:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T00:24:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]': finished 2024-09-21T00:24:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]': finished 2024-09-21T00:24:07.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]': finished 2024-09-21T00:24:07.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T00:24:07.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:07 smithi031 ceph-mon[63863]: osd.7 marked itself down and dead 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]': finished 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]': finished 2024-09-21T00:24:07.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[78227]: osd.7 marked itself down and dead 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.16"}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1a"}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1e"}]': finished 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T00:24:07.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:07 smithi018 ceph-mon[80555]: osd.7 marked itself down and dead 2024-09-21T00:24:08.047 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:07 smithi031 podman[87800]: 2024-09-21 00:24:07.791720222 +0000 UTC m=+0.994948444 container died c0a826304bb31bcd47b47e8609251e4224a3ff0790d37482b0c72ddf08f2a12b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, ceph=True) 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: pgmap v254: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T00:24:08.313 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:08 smithi031 ceph-mon[63863]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T00:24:08.313 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 podman[87800]: 2024-09-21 00:24:08.135647111 +0000 UTC m=+1.338875332 container cleanup c0a826304bb31bcd47b47e8609251e4224a3ff0790d37482b0c72ddf08f2a12b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., ceph=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8) 2024-09-21T00:24:08.313 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 bash[87800]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: pgmap v254: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[80555]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: pgmap v254: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T00:24:08.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T00:24:08.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T00:24:08.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T00:24:08.676 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:08 smithi018 ceph-mon[78227]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T00:24:08.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 podman[87815]: 2024-09-21 00:24:08.312122169 +0000 UTC m=+0.518826332 container remove c0a826304bb31bcd47b47e8609251e4224a3ff0790d37482b0c72ddf08f2a12b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, distribution-scope=public, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, release=754, com.redhat.component=centos-stream-container) 2024-09-21T00:24:09.083 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 podman[87907]: 2024-09-21 00:24:08.668341131 +0000 UTC m=+0.016533916 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:09.083 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 podman[87907]: 2024-09-21 00:24:08.77138806 +0000 UTC m=+0.119580886 container create 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:24:09.083 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:08 smithi031 podman[87907]: 2024-09-21 00:24:08.998882325 +0000 UTC m=+0.347075110 container init 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T00:24:09.084 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 podman[87907]: 2024-09-21 00:24:09.002470941 +0000 UTC m=+0.350663725 container start 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T00:24:09.084 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 podman[87907]: 2024-09-21 00:24:09.082421572 +0000 UTC m=+0.430614360 container attach 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T00:24:09.486 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 podman[87907]: 2024-09-21 00:24:09.203363308 +0000 UTC m=+0.551556101 container died 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2) 2024-09-21T00:24:10.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:09 smithi018 ceph-mon[78227]: osdmap e157: 8 total, 7 up, 8 in 2024-09-21T00:24:10.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:09 smithi018 ceph-mon[78227]: pgmap v258: 161 pgs: 23 stale+active+clean, 3 peering, 135 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:09 smithi018 ceph-mon[80555]: osdmap e157: 8 total, 7 up, 8 in 2024-09-21T00:24:10.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:09 smithi018 ceph-mon[80555]: pgmap v258: 161 pgs: 23 stale+active+clean, 3 peering, 135 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:10.195 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:09 smithi031 ceph-mon[63863]: osdmap e157: 8 total, 7 up, 8 in 2024-09-21T00:24:10.195 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:09 smithi031 ceph-mon[63863]: pgmap v258: 161 pgs: 23 stale+active+clean, 3 peering, 135 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:10.195 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 podman[87931]: 2024-09-21 00:24:09.753635688 +0000 UTC m=+0.544102021 container remove 46514ae34337ec9abb65848a9e340f49370316526dc52c3d189f7c6ac1f5e731 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70) 2024-09-21T00:24:10.195 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T00:24:10.195 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 systemd[1]: Stopped Ceph osd.7 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:24:10.195 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.7.service: Consumed 18.951s CPU time. 2024-09-21T00:24:10.195 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:09 smithi031 systemd[1]: Starting Ceph osd.7 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:24:10.499 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 podman[88032]: 2024-09-21 00:24:10.097205396 +0000 UTC m=+0.016610574 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:10.499 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 podman[88032]: 2024-09-21 00:24:10.273541692 +0000 UTC m=+0.192946877 container create d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T00:24:10.499 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 podman[88032]: 2024-09-21 00:24:10.498854618 +0000 UTC m=+0.418259804 container init d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 podman[88032]: 2024-09-21 00:24:10.50263759 +0000 UTC m=+0.422042771 container start d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 podman[88032]: 2024-09-21 00:24:10.59900855 +0000 UTC m=+0.518413744 container attach d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 bash[88032]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:24:10.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:10 smithi031 bash[88032]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T00:24:11.114 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:10 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:11.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:10 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:11.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:10 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:11.406 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T00:24:11.406 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T00:24:11.407 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T00:24:11.407 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T00:24:11.407 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T00:24:11.407 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T00:24:11.721 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate[88048]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T00:24:11.722 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 bash[88032]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T00:24:11.722 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:11 smithi031 podman[88032]: 2024-09-21 00:24:11.436939876 +0000 UTC m=+1.356345056 container died d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T00:24:12.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:11 smithi031 ceph-mon[63863]: pgmap v259: 161 pgs: 2 active+clean+remapped, 32 active+undersized, 10 stale+active+clean, 3 peering, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 59/723 objects degraded (8.160%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:12.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:11 smithi018 ceph-mon[78227]: pgmap v259: 161 pgs: 2 active+clean+remapped, 32 active+undersized, 10 stale+active+clean, 3 peering, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 59/723 objects degraded (8.160%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:12.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:11 smithi018 ceph-mon[80555]: pgmap v259: 161 pgs: 2 active+clean+remapped, 32 active+undersized, 10 stale+active+clean, 3 peering, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 59/723 objects degraded (8.160%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:11] "GET /metrics HTTP/1.1" 200 38237 "" "Prometheus/2.43.0" 2024-09-21T00:24:12.344 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 podman[88184]: 2024-09-21 00:24:12.002882063 +0000 UTC m=+0.559736982 container remove d8ebbfa9e4a832a41e788f51b547aba261f8f8b7bd8b4a370596ce5cc3f1e45c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:24:12.344 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 podman[88245]: 2024-09-21 00:24:12.124966978 +0000 UTC m=+0.017295584 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:12.344 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 podman[88245]: 2024-09-21 00:24:12.245409425 +0000 UTC m=+0.137738008 container create 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:24:12.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 podman[88245]: 2024-09-21 00:24:12.464487956 +0000 UTC m=+0.356816553 container init 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T00:24:12.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 podman[88245]: 2024-09-21 00:24:12.467877489 +0000 UTC m=+0.360206073 container start 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T00:24:12.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 ceph-osd[88264]: -- 172.21.15.31:0/325337701 <== mon.2 v2:172.21.15.31:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56550b13ba40 con 0x56550bf02000 2024-09-21T00:24:12.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 bash[88245]: 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 2024-09-21T00:24:12.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:12 smithi031 systemd[1]: Started Ceph osd.7 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:24:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[78227]: Health check failed: Degraded data redundancy: 59/723 objects degraded (8.160%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:13.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:13.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[80555]: Health check failed: Degraded data redundancy: 59/723 objects degraded (8.160%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:13.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:12 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:13.204 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:12 smithi031 ceph-mon[63863]: Health check failed: Degraded data redundancy: 59/723 objects degraded (8.160%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:13.204 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:12 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:13.204 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:12 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:14.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:13 smithi018 ceph-mon[78227]: pgmap v260: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:14.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:13 smithi018 ceph-mon[80555]: pgmap v260: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:14.180 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:13 smithi031 ceph-mon[63863]: pgmap v260: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:14.447 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:14 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:24:14.370+0000 7ff223864640 -1 Falling back to public interface 2024-09-21T00:24:15.125 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:14 smithi031 ceph-mon[63863]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T00:24:15.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:14 smithi018 ceph-mon[78227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T00:24:15.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:14 smithi018 ceph-mon[80555]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T00:24:16.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:15 smithi018 ceph-mon[78227]: pgmap v261: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:16.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:15 smithi018 ceph-mon[80555]: pgmap v261: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:16.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:15 smithi031 ceph-mon[63863]: pgmap v261: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:18.079 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:24:17.884+0000 7ff223864640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-21T00:24:18.079 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:24:17.912+0000 7ff223864640 -1 osd.7 154 log_to_monitors true 2024-09-21T00:24:18.079 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-mon[63863]: pgmap v262: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:18.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.080 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:17 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[78227]: pgmap v262: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[80555]: pgmap v262: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:18.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:18.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:17 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:19.035 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:18 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:19.035 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:18 smithi031 ceph-mon[63863]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.035 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:18 smithi031 ceph-mon[63863]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.075 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[80555]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[80555]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 98/723 objects degraded (13.555%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:19.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[78227]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:18 smithi018 ceph-mon[78227]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T00:24:19.339 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:24:19.221+0000 7ff217607640 -1 osd.7 154 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T00:24:20.077 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T00:24:20.077 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: osdmap e158: 8 total, 7 up, 8 in 2024-09-21T00:24:20.077 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.078 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.078 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: pgmap v264: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:20.078 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='osd.7 ' entity='osd.7' 2024-09-21T00:24:20.078 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:20.078 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: osdmap e158: 8 total, 7 up, 8 in 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: pgmap v264: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:20.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='osd.7 ' entity='osd.7' 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: osdmap e158: 8 total, 7 up, 8 in 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi031", "root=default"]}]: dispatch 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: pgmap v264: 161 pgs: 2 active+clean+remapped, 42 active+undersized, 25 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 98/723 objects degraded (13.555%); 0 B/s, 0 objects/s recovering 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='osd.7 ' entity='osd.7' 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:20.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[80555]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[80555]: osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744] boot 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[80555]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:21.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[78227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:24:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[78227]: osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744] boot 2024-09-21T00:24:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[78227]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T00:24:21.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:24:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:20 smithi031 ceph-mon[63863]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T00:24:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:20 smithi031 ceph-mon[63863]: osd.7 [v2:172.21.15.31:6824/1354398744,v1:172.21.15.31:6825/1354398744] boot 2024-09-21T00:24:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:20 smithi031 ceph-mon[63863]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T00:24:21.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:21] "GET /metrics HTTP/1.1" 200 38172 "" "Prometheus/2.43.0" 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: pgmap v266: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: pgmap v267: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T00:24:22.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: pgmap v266: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: pgmap v267: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-21T00:24:22.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-21T00:24:22.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nfs 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-21T00:24:22.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nfs 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: Upgrade: Finalizing container_image settings 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:24:22.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:24:22.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: Upgrade: Finalizing container_image settings 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:24:22.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:24:22.187 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.190 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:21 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: pgmap v266: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: pgmap v267: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-21T00:24:22.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-21T00:24:22.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T00:24:22.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nfs 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:24:22.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:24:22.257 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:24:22.258 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:22.259 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:21 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: Upgrade: Complete! 2024-09-21T00:24:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: pgmap v269: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: pgmap v270: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: pgmap v271: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: pgmap v272: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[78227]: Health check update: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: Upgrade: Complete! 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: pgmap v269: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: pgmap v270: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: pgmap v271: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: pgmap v272: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T00:24:23.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:23 smithi018 ceph-mon[80555]: Health check update: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:24:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: pgmap v269: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: pgmap v270: 161 pgs: 12 peering, 2 active+clean+remapped, 34 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 81/723 objects degraded (11.203%) 2024-09-21T00:24:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: pgmap v271: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: pgmap v272: 161 pgs: 12 peering, 2 active+clean+remapped, 33 active+undersized, 18 active+undersized+degraded, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:23.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T00:24:23.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:23 smithi031 ceph-mon[63863]: Health check update: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T00:24:23.563 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:24:24.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:24 smithi018 ceph-mon[78227]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:24.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:24 smithi018 ceph-mon[80555]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:24.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:24 smithi031 ceph-mon[63863]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:25.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:25 smithi018 ceph-mon[78227]: pgmap v274: 161 pgs: 12 peering, 2 active+clean+remapped, 30 active+undersized, 18 active+undersized+degraded, 99 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:25.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:25 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:25.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:25 smithi018 ceph-mon[80555]: pgmap v274: 161 pgs: 12 peering, 2 active+clean+remapped, 30 active+undersized, 18 active+undersized+degraded, 99 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:25.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:25 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:25.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:25 smithi031 ceph-mon[63863]: pgmap v274: 161 pgs: 12 peering, 2 active+clean+remapped, 30 active+undersized, 18 active+undersized+degraded, 99 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-09-21T00:24:25.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:25 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (21m) 2m ago 29m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (8m) 8s ago 29m 76.9M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 2m ago 28m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (8m) 8s ago 34m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (18m) 2m ago 35m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (6m) 2m ago 35m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:24:25.512 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (7m) 8s ago 34m 49.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (6m) 2m ago 34m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (21m) 2m ago 30m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (21m) 8s ago 30m 16.2M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 2m ago 33m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (4m) 2m ago 33m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 2m ago 33m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (2m) 2m ago 32m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (115s) 8s ago 32m 48.0M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (81s) 8s ago 31m 44.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (46s) 8s ago 31m 44.6M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (13s) 8s ago 31m 12.2M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (8m) 8s ago 30m 69.3M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (28m) 2m ago 28m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:24:25.513 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (28m) 8s ago 28m 82.2M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:24:26.184 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-09-21T00:24:26.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:26 smithi018 ceph-mon[78227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded) 2024-09-21T00:24:26.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:26 smithi018 ceph-mon[80555]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded) 2024-09-21T00:24:26.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:26 smithi031 ceph-mon[63863]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/723 objects degraded (9.129%), 18 pgs degraded) 2024-09-21T00:24:27.565 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:27 smithi018 ceph-mon[80555]: pgmap v275: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:24:27.565 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:27 smithi018 ceph-mon[80555]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:27.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:27 smithi018 ceph-mon[78227]: pgmap v275: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:24:27.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:27 smithi018 ceph-mon[78227]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:27.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:27 smithi031 ceph-mon[63863]: pgmap v275: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:24:27.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:27 smithi031 ceph-mon[63863]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:28.361 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:24:29.086 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-09-21T00:24:29.401 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:29 smithi018 ceph-mon[78227]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:24:29.401 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:29 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2219175838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:29.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:29 smithi018 ceph-mon[80555]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:24:29.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:29 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2219175838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:29.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:29 smithi031 ceph-mon[63863]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-21T00:24:29.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:29 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2219175838' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:31.435 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-21T00:24:31.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:31 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:31.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:31 smithi018 ceph-mon[78227]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:24:31.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:31 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:31.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:31 smithi018 ceph-mon[80555]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:24:31.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:31 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:31.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:31 smithi031 ceph-mon[63863]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:24:32.028 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:31] "GET /metrics HTTP/1.1" 200 38172 "" "Prometheus/2.43.0" 2024-09-21T00:24:32.057 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:24:32.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:32 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3696504892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:32.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:32 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3696504892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:32.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:32 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3696504892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:33.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:33 smithi018 ceph-mon[78227]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:33 smithi018 ceph-mon[80555]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:33.652 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:33 smithi031 ceph-mon[63863]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:24:34.259 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:24:34.259 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:24:34.260 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:24:34.927 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:24:35.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[78227]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[78227]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[80555]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[80555]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:35.505 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:35.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:35 smithi031 ceph-mon[63863]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-09-21T00:24:35.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:35 smithi031 ceph-mon[63863]: from='client.44604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:35.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:35.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:35.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:37.067 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:24:37.067 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:24:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:24:37.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:37 smithi018 ceph-mon[78227]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:37.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:37 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1038210822' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:24:37.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:37 smithi018 ceph-mon[80555]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:37.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:37 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1038210822' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:24:37.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-09-21T00:24:37.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:37 smithi031 ceph-mon[63863]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:37.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:37 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1038210822' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:24:39.674 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:39 smithi018 ceph-mon[78227]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:39.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:39 smithi018 ceph-mon[80555]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:39.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:39 smithi031 ceph-mon[63863]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:24:41.123 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:41.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:24:41.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:41 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:41.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T00:24:42.134 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:41] "GET /metrics HTTP/1.1" 200 38323 "" "Prometheus/2.43.0" 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[78227]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[78227]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[78227]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[78227]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[80555]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[80555]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[80555]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:42.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:42 smithi018 ceph-mon[80555]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:42.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:42 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:42.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:42 smithi031 ceph-mon[63863]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:24:42.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:42 smithi031 ceph-mon[63863]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:42.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:42 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mgr 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:44.328 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mgr 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:24:44.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:44.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:44 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.382 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.383 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all osd 2024-09-21T00:24:44.384 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:44.384 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:44.384 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mds 2024-09-21T00:24:44.384 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:44 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:45.972 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:24:46.435 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: Upgrade: Updating rgw.foo.smithi018.kionxd (1/2) 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[80555]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: Upgrade: Updating rgw.foo.smithi018.kionxd (1/2) 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:46.436 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:46.437 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:46 smithi018 ceph-mon[78227]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: Upgrade: Updating rgw.foo.smithi018.kionxd (1/2) 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi018.kionxd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:46.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:46 smithi031 ceph-mon[63863]: Deploying daemon rgw.foo.smithi018.kionxd on smithi018 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (21m) 2m ago 29m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (8m) 29s ago 29m 76.9M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 2m ago 29m 63.5M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (8m) 29s ago 34m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (18m) 2m ago 36m 503M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (6m) 2m ago 36m 53.7M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (7m) 29s ago 34m 49.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (6m) 2m ago 34m 42.4M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (22m) 2m ago 30m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (21m) 29s ago 30m 16.2M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:24:46.676 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (5m) 2m ago 34m 45.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (4m) 2m ago 33m 53.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 2m ago 33m 42.6M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (2m) 2m ago 33m 12.1M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (2m) 29s ago 32m 48.0M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (103s) 29s ago 32m 44.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (68s) 29s ago 31m 44.6M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (34s) 29s ago 31m 12.2M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (9m) 29s ago 30m 69.3M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (29m) 2m ago 29m 80.5M - 17.2.0 e1d6a67b021e 4390ae8ed52c 2024-09-21T00:24:46.677 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (29m) 29s ago 29m 82.2M - 17.2.0 e1d6a67b021e b66c1930c503 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:24:47.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 13 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:24:47.124 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:24:47.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:47 smithi018 ceph-mon[80555]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:47.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:47 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/339420152' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:47.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:47 smithi018 ceph-mon[78227]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:47.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:47 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/339420152' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:47.496 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:24:47.496 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:24:47.497 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:24:47.497 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-09-21T00:24:47.497 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:24:47.498 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/2 daemons upgraded", 2024-09-21T00:24:47.498 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading rgw daemons", 2024-09-21T00:24:47.498 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:24:47.498 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:24:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:47 smithi031 ceph-mon[63863]: from='client.44622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:47 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/339420152' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:24:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:48 smithi031 ceph-mon[63863]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:48 smithi031 ceph-mon[63863]: from='client.54568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:48 smithi031 ceph-mon[63863]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:48.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[78227]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[78227]: from='client.54568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[78227]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:48.599 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[80555]: from='client.44628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.599 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[80555]: from='client.54568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:48.599 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:48 smithi018 ceph-mon[80555]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:49.421 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:49 smithi018 ceph-mon[80555]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:49.422 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:49 smithi018 ceph-mon[78227]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:49 smithi031 ceph-mon[63863]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:24:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:50 smithi031 ceph-mon[63863]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:50.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[80555]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:50.677 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[78227]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:50.678 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:24:51.326 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:51 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:51.326 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:51.326 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:51.326 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:51 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:51.656 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:51.656 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:51.656 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:51.656 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:51.656 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:24:51.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:51.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi031.jzvksl", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T00:24:51.657 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:24:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:24:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:24:51] "GET /metrics HTTP/1.1" 200 38323 "" "Prometheus/2.43.0" 2024-09-21T00:24:52.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:52 smithi031 ceph-mon[63863]: Upgrade: Updating rgw.foo.smithi031.jzvksl (2/2) 2024-09-21T00:24:52.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:52 smithi031 ceph-mon[63863]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:24:52.380 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:52 smithi031 ceph-mon[63863]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 102 B/s wr, 99 op/s 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[80555]: Upgrade: Updating rgw.foo.smithi031.jzvksl (2/2) 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[80555]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[80555]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 102 B/s wr, 99 op/s 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[78227]: Upgrade: Updating rgw.foo.smithi031.jzvksl (2/2) 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[78227]: Deploying daemon rgw.foo.smithi031.jzvksl on smithi031 2024-09-21T00:24:52.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:52 smithi018 ceph-mon[78227]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 102 B/s wr, 99 op/s 2024-09-21T00:24:54.411 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:54 smithi031 ceph-mon[63863]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 85 B/s wr, 102 op/s 2024-09-21T00:24:54.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:54 smithi018 ceph-mon[78227]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 85 B/s wr, 102 op/s 2024-09-21T00:24:54.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:54 smithi018 ceph-mon[80555]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 85 B/s wr, 102 op/s 2024-09-21T00:24:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:55 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:55 smithi031 ceph-mon[63863]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 172 op/s 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[78227]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 172 op/s 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:55.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:55 smithi018 ceph-mon[80555]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 172 op/s 2024-09-21T00:24:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:58 smithi031 ceph-mon[63863]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 170 B/s wr, 215 op/s 2024-09-21T00:24:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:58 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:58.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:58 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:58.674 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[78227]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 170 B/s wr, 215 op/s 2024-09-21T00:24:58.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:58.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:58.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[80555]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 137 KiB/s rd, 170 B/s wr, 215 op/s 2024-09-21T00:24:58.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:58.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:58 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.714 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.714 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 234 op/s 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 234 op/s 2024-09-21T00:24:59.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:24:59.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:24:59 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 234 op/s 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:24:59 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:00.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:00 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:00 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:00 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:01.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:01 smithi031 ceph-mon[63863]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-09-21T00:25:01.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:01 smithi018 ceph-mon[78227]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-09-21T00:25:01.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:01 smithi018 ceph-mon[80555]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-09-21T00:25:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:01] "GET /metrics HTTP/1.1" 200 38323 "" "Prometheus/2.43.0" 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]': finished 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]: dispatch 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]: dispatch 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]': finished 2024-09-21T00:25:02.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:02 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]: dispatch 2024-09-21T00:25:03.152 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.153 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:03.154 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]: dispatch 2024-09-21T00:25:03.155 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi018.kionxd"}]': finished 2024-09-21T00:25:03.155 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:02 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]: dispatch 2024-09-21T00:25:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 88 B/s wr, 172 op/s 2024-09-21T00:25:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 107 B/s wr, 183 op/s 2024-09-21T00:25:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:04.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]': finished 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nfs 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Finalizing container_image settings 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 88 B/s wr, 172 op/s 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 107 B/s wr, 183 op/s 2024-09-21T00:25:04.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]': finished 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nfs 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Finalizing container_image settings 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:25:04.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.181 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Upgrade: Complete! 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 116 op/s 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 68 op/s 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: Checking dashboard <-> RGW credentials 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.182 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 38 op/s 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:25:04.183 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Upgrade: Complete! 2024-09-21T00:25:04.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 116 op/s 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 68 op/s 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: Checking dashboard <-> RGW credentials 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 38 op/s 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:25:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:03 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 88 B/s wr, 172 op/s 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 107 B/s wr, 183 op/s 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi031.jzvksl"}]': finished 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nfs 2024-09-21T00:25:04.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all nvmeof 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Finalizing container_image settings 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T00:25:04.253 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T00:25:04.254 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T00:25:04.255 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Upgrade: Complete! 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 116 op/s 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 68 op/s 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: Checking dashboard <-> RGW credentials 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 38 op/s 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:25:04.256 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:03 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:05.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:05.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:05.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:05.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:06.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:05 smithi018 ceph-mon[78227]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6 op/s 2024-09-21T00:25:06.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:05 smithi018 ceph-mon[80555]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6 op/s 2024-09-21T00:25:06.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:05 smithi031 ceph-mon[63863]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6 op/s 2024-09-21T00:25:08.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:08 smithi018 ceph-mon[78227]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-21T00:25:08.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:08 smithi018 ceph-mon[80555]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-21T00:25:08.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:08 smithi031 ceph-mon[63863]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-21T00:25:09.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:09 smithi018 ceph-mon[78227]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-09-21T00:25:09.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:09 smithi018 ceph-mon[80555]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-09-21T00:25:09.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:09 smithi031 ceph-mon[63863]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-09-21T00:25:10.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:10 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:10.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:10 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:10.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:10 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:11.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:11 smithi018 ceph-mon[78227]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-21T00:25:11.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:11 smithi018 ceph-mon[80555]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-21T00:25:11.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:11 smithi031 ceph-mon[63863]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-21T00:25:12.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:11] "GET /metrics HTTP/1.1" 200 38355 "" "Prometheus/2.43.0" 2024-09-21T00:25:14.674 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:14 smithi018 ceph-mon[80555]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-21T00:25:14.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:14 smithi018 ceph-mon[78227]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-21T00:25:14.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:14 smithi031 ceph-mon[63863]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-21T00:25:15.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:15 smithi018 ceph-mon[78227]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-21T00:25:15.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:15 smithi018 ceph-mon[80555]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-21T00:25:15.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:15 smithi031 ceph-mon[63863]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-21T00:25:18.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:18 smithi018 ceph-mon[78227]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:25:18.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:18 smithi018 ceph-mon[80555]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:25:18.554 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:25:18.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:18 smithi031 ceph-mon[63863]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-09-21T00:25:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:19 smithi018 ceph-mon[78227]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:19 smithi018 ceph-mon[78227]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:19 smithi018 ceph-mon[80555]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:19 smithi018 ceph-mon[80555]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:19.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:19 smithi031 ceph-mon[63863]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:19.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:19 smithi031 ceph-mon[63863]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:20.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:20 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:20.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:20 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:20.489 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (22m) 21s ago 30m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (9m) 22s ago 30m 80.5M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 21s ago 29m 63.6M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (9m) 22s ago 35m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (19m) 21s ago 36m 508M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (7m) 21s ago 36m 62.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (8m) 22s ago 35m 51.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (7m) 21s ago 35m 49.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (22m) 21s ago 31m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (22m) 22s ago 31m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (5m) 21s ago 34m 54.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:25:20.490 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 21s ago 34m 63.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (4m) 21s ago 34m 49.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (3m) 21s ago 33m 61.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (2m) 22s ago 33m 52.4M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (2m) 22s ago 32m 48.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (101s) 22s ago 32m 48.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (68s) 22s ago 32m 57.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (9m) 22s ago 31m 71.4M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (31s) 21s ago 29m 76.8M - 19.1.1-367-g9e2ac70b cc9251b744a3 6a70459e4967 2024-09-21T00:25:20.491 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (26s) 22s ago 29m 78.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 7f93ad9844ef 2024-09-21T00:25:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:20.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:20 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:21.079 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-09-21T00:25:21.665 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:21 smithi018 ceph-mon[78227]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:21.665 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:21 smithi018 ceph-mon[78227]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:21.665 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:21 smithi018 ceph-mon[80555]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:21.666 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:21 smithi018 ceph-mon[80555]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:21.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:21 smithi031 ceph-mon[63863]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:21.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:21 smithi031 ceph-mon[63863]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:21] "GET /metrics HTTP/1.1" 200 38352 "" "Prometheus/2.43.0" 2024-09-21T00:25:23.108 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:25:23.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:23 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2593366086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:23.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:23 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2593366086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:23 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2593366086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:23.752 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-09-21T00:25:24.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:24 smithi018 ceph-mon[78227]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:24.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:24 smithi018 ceph-mon[80555]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:24.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:24 smithi031 ceph-mon[63863]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:25.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:25 smithi018 ceph-mon[78227]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:25.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:25 smithi018 ceph-mon[80555]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:25.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:25 smithi031 ceph-mon[63863]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:25.792 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-21T00:25:26.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:25:26.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:26 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3245401597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:26.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:26 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3245401597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:26.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:26 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3245401597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:27.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:27 smithi018 ceph-mon[78227]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:27.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:27 smithi018 ceph-mon[80555]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:27.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:27 smithi031 ceph-mon[63863]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T00:25:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T00:25:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:25:28.491 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:25:29.201 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:25:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:29 smithi018 ceph-mon[78227]: from='client.44754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:29.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:29 smithi018 ceph-mon[78227]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:29 smithi018 ceph-mon[80555]: from='client.44754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:29.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:29 smithi018 ceph-mon[80555]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:29.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:29 smithi031 ceph-mon[63863]: from='client.44754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:29.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:29 smithi031 ceph-mon[63863]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:30.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:30 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:30.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:30 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:30.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:30 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:31.231 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:25:31.232 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:25:31.232 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:25:31.587 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:31 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/2487194245' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:31.587 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:31 smithi018 ceph-mon[78227]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:31.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:31 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/2487194245' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:31.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:31 smithi018 ceph-mon[80555]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:31.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:31 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/2487194245' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:31.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:31 smithi031 ceph-mon[63863]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:32.047 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T00:25:32.355 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:31] "GET /metrics HTTP/1.1" 200 38352 "" "Prometheus/2.43.0" 2024-09-21T00:25:33.635 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:33 smithi018 ceph-mon[78227]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:33.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:33 smithi018 ceph-mon[80555]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:33.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:33 smithi031 ceph-mon[63863]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:34.119 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:34.843 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T00:25:34.853 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T00:25:34.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:35.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 943 B/s rd, 0 op/s 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 943 B/s rd, 0 op/s 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.397 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 943 B/s rd, 0 op/s 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:25:35.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:37.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:25:37.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:25:37.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mgr 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mon 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all crash 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all osd 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all mds 2024-09-21T00:25:37.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[80555]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mgr 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mon 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all crash 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all osd 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all mds 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:37.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:37 smithi018 ceph-mon[78227]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Target is version 19.1.1-367-g9e2ac70b (squid) 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, digests ['quay.ceph.io/ceph-ci/ceph@sha256:07c2c7a2cb9ca9b41c609b2ddfb8539abdc0283fbb4aebf87c70eeaf670bffe5'] 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mgr 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mon 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all crash 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all osd 2024-09-21T00:25:37.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all mds 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rgw 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:37.503 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:37 smithi031 ceph-mon[63863]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:39.058 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:25:39.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[78227]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:39.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[80555]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:25:39.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:39 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:39 smithi031 ceph-mon[63863]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi018.wbwhhs", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-21T00:25:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:39 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (22m) 41s ago 30m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (9m) 41s ago 30m 80.5M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 41s ago 29m 63.6M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (9m) 41s ago 35m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (19m) 41s ago 37m 508M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (7m) 41s ago 37m 62.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (8m) 41s ago 35m 51.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (7m) 41s ago 35m 49.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (22m) 41s ago 31m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (22m) 41s ago 31m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 41s ago 35m 54.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 41s ago 34m 63.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:25:39.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (4m) 41s ago 34m 49.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (3m) 41s ago 33m 61.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (3m) 41s ago 33m 52.4M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (2m) 41s ago 33m 48.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (2m) 41s ago 32m 48.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (87s) 41s ago 32m 57.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (9m) 41s ago 31m 71.4M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (51s) 41s ago 30m 76.8M - 19.1.1-367-g9e2ac70b cc9251b744a3 6a70459e4967 2024-09-21T00:25:39.764 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (45s) 41s ago 30m 78.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 7f93ad9844ef 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:25:40.185 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 15 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:25:40.186 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:25:40.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[78227]: Upgrade: Updating iscsi.foo.smithi018.wbwhhs 2024-09-21T00:25:40.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[78227]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:25:40.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[78227]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[80555]: Upgrade: Updating iscsi.foo.smithi018.wbwhhs 2024-09-21T00:25:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[80555]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:25:40.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:40 smithi018 ceph-mon[80555]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:40 smithi031 ceph-mon[63863]: Upgrade: Updating iscsi.foo.smithi018.wbwhhs 2024-09-21T00:25:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:40 smithi031 ceph-mon[63863]: Deploying daemon iscsi.foo.smithi018.wbwhhs on smithi018 2024-09-21T00:25:40.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:40 smithi031 ceph-mon[63863]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw", 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "osd", 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T00:25:40.561 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "15/21 daemons upgraded", 2024-09-21T00:25:40.562 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading iscsi daemons", 2024-09-21T00:25:40.562 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T00:25:40.562 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:25:41.018 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-21T00:25:41.018 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:25:41.018 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:25:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: from='client.44784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-09-21T00:25:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3145288124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:41.502 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:41 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3302551349' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: from='client.44784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3145288124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3302551349' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:41.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: from='client.44784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:41.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-09-21T00:25:41.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3145288124' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:25:41.676 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3302551349' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:25:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:41] "GET /metrics HTTP/1.1" 200 38355 "" "Prometheus/2.43.0" 2024-09-21T00:25:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:42 smithi031 ceph-mon[63863]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:42.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:42 smithi018 ceph-mon[78227]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:42.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:42 smithi018 ceph-mon[80555]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:25:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:43 smithi031 ceph-mon[63863]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:43.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:43 smithi018 ceph-mon[78227]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:43.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:43 smithi018 ceph-mon[80555]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:45 smithi031 ceph-mon[63863]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:25:45.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:45 smithi018 ceph-mon[78227]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:25:45.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:45 smithi018 ceph-mon[80555]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:25:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:47 smithi031 ceph-mon[63863]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:47.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:47 smithi018 ceph-mon[78227]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:47.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:47 smithi018 ceph-mon[80555]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:25:49.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:49 smithi018 ceph-mon[78227]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:49.477 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:49 smithi018 ceph-mon[80555]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:49 smithi031 ceph-mon[63863]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:50.490 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:50.490 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:25:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:51 smithi031 ceph-mon[63863]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:51.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:51 smithi031 ceph-mon[63863]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:51.594 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:25:51.450+0000 7fd043705640 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:25:51.594 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-mon[80555]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:51.594 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-mon[80555]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:51.595 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-mon[78227]: from='client.25070 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:25:51.595 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-mon[78227]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:25:52.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:25:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:25:51] "GET /metrics HTTP/1.1" 200 38353 "" "Prometheus/2.43.0" 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:52.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:52.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:25:52.752 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:52.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-21T00:25:52.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:25:53.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:53 smithi031 ceph-mon[63863]: Health check failed: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-21T00:25:53.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:53 smithi018 ceph-mon[78227]: Health check failed: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-21T00:25:53.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:53 smithi018 ceph-mon[80555]: Health check failed: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-21T00:25:54.751 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:54 smithi031 ceph-mon[63863]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:54.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:54 smithi018 ceph-mon[78227]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:54.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:54 smithi018 ceph-mon[80555]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:56.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:56 smithi018 ceph-mon[78227]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:56.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:56 smithi018 ceph-mon[80555]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:57.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:56 smithi031 ceph-mon[63863]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:58.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:25:58 smithi018 ceph-mon[78227]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:58.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:25:58 smithi018 ceph-mon[80555]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:25:59.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:25:58 smithi031 ceph-mon[63863]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s 2024-09-21T00:26:00.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:00 smithi018 ceph-mon[78227]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:00.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:00 smithi018 ceph-mon[80555]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:01.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:00 smithi031 ceph-mon[63863]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:01] "GET /metrics HTTP/1.1" 200 38353 "" "Prometheus/2.43.0" 2024-09-21T00:26:02.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:02 smithi018 ceph-mon[80555]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:02.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:02 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/70876719' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:26:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:02 smithi018 ceph-mon[78227]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:02.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:02 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/70876719' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:26:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:02 smithi031 ceph-mon[63863]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:03.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:02 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/70876719' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-21T00:26:03.903 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:03 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]: dispatch 2024-09-21T00:26:03.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:03 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]: dispatch 2024-09-21T00:26:03.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:03 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]: dispatch 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]': finished 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3064116823' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:04.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]': finished 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3064116823' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:04.926 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:04 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3443216277' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/2970699224"}]': finished 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3064116823' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]: dispatch 2024-09-21T00:26:05.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:05.002 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:04 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:05.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]': finished 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[80555]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/7690836' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]': finished 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[78227]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/7690836' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:05.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:05 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:05 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/1674040012"}]': finished 2024-09-21T00:26:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:05 smithi031 ceph-mon[63863]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T00:26:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:05 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/7690836' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:06.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:05 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]: dispatch 2024-09-21T00:26:06.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[78227]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]': finished 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[78227]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]: dispatch 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[80555]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]': finished 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[80555]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T00:26:06.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:06 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]: dispatch 2024-09-21T00:26:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:06 smithi031 ceph-mon[63863]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-09-21T00:26:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:06 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/3411271810"}]': finished 2024-09-21T00:26:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:06 smithi031 ceph-mon[63863]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T00:26:07.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:06 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]: dispatch 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]': finished 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[80555]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]: dispatch 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]': finished 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[78227]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T00:26:07.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:07 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]: dispatch 2024-09-21T00:26:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:07 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3018387482' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6800/1853405495"}]': finished 2024-09-21T00:26:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:07 smithi031 ceph-mon[63863]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T00:26:08.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:07 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]: dispatch 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[78227]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]': finished 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[78227]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3270693916' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[80555]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]': finished 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[80555]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3270693916' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:08.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:08 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:08 smithi031 ceph-mon[63863]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:08 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1600422772' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:6801/1853405495"}]': finished 2024-09-21T00:26:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:08 smithi031 ceph-mon[63863]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T00:26:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:08 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3270693916' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:09.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:08 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]: dispatch 2024-09-21T00:26:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:09 smithi018 ceph-mon[78227]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]': finished 2024-09-21T00:26:09.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:09 smithi018 ceph-mon[78227]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T00:26:09.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:09 smithi018 ceph-mon[80555]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]': finished 2024-09-21T00:26:09.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:09 smithi018 ceph-mon[80555]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T00:26:10.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:09 smithi031 ceph-mon[63863]: from='client.? ' entity='client.iscsi.foo.smithi018.wbwhhs' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.18:0/209570323"}]': finished 2024-09-21T00:26:10.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:09 smithi031 ceph-mon[63863]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T00:26:10.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:10 smithi018 ceph-mon[78227]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:10.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:10 smithi018 ceph-mon[80555]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:11.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:10 smithi031 ceph-mon[63863]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-09-21T00:26:11.401 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:26:11.752 INFO:teuthology.orchestra.run.smithi018.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed." 2024-09-21T00:26:12.045 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:11] "GET /metrics HTTP/1.1" 200 38431 "" "Prometheus/2.43.0" 2024-09-21T00:26:12.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:26:12.658 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[78227]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:12.658 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[78227]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:26:12.658 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[78227]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:12.659 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:12.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[80555]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:12.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[80555]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:26:12.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[80555]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:12.659 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:12 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:13.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:12 smithi031 ceph-mon[63863]: from='client.44868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:13.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:12 smithi031 ceph-mon[63863]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-21T00:26:13.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:12 smithi031 ceph-mon[63863]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:13.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:12 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:14.251 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:26:13 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=cleanup t=2024-09-21T00:26:13.766407025Z level=info msg="Completed cleanup jobs" duration=4.670928ms 2024-09-21T00:26:14.346 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:26:14.346 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (23m) 75s ago 31m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:26:14.346 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (10m) 76s ago 31m 80.5M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:26:14.346 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 75s ago 30m 63.6M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (10m) 76s ago 36m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (20m) 75s ago 37m 508M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (8m) 75s ago 37m 62.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (9m) 76s ago 36m 51.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (7m) 75s ago 36m 49.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (23m) 75s ago 32m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (23m) 76s ago 32m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 75s ago 35m 54.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (6m) 75s ago 35m 63.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (5m) 75s ago 34m 49.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (4m) 75s ago 34m 61.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (3m) 76s ago 34m 52.4M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (3m) 76s ago 33m 48.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:26:14.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (2m) 76s ago 33m 48.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:26:14.348 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (2m) 76s ago 32m 57.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:26:14.348 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (10m) 76s ago 32m 71.4M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:26:14.348 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (85s) 75s ago 30m 76.8M - 19.1.1-367-g9e2ac70b cc9251b744a3 6a70459e4967 2024-09-21T00:26:14.348 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (80s) 76s ago 30m 78.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 7f93ad9844ef 2024-09-21T00:26:14.675 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:14 smithi018 ceph-mon[78227]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-09-21T00:26:14.675 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:14 smithi018 ceph-mon[80555]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-09-21T00:26:15.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:14 smithi031 ceph-mon[63863]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-09-21T00:26:15.081 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:26:15.673 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:15 smithi018 ceph-mon[80555]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:15.673 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:15 smithi018 ceph-mon[78227]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:16.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:15 smithi031 ceph-mon[63863]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:26:16.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:16 smithi018 ceph-mon[78227]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:26:16.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:16 smithi018 ceph-mon[80555]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:26:17.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:16 smithi031 ceph-mon[63863]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-21T00:26:17.133 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:26:17.134 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 15 2024-09-21T00:26:17.135 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:26:17.135 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:26:17.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:17 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3686053476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:26:17.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:17 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3686053476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:26:17.736 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-21T00:26:18.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:17 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3686053476' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:26:18.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:18 smithi018 ceph-mon[78227]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:26:18.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:18 smithi018 ceph-mon[80555]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:26:19.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:18 smithi031 ceph-mon[63863]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-21T00:26:19.512 INFO:teuthology.orchestra.run.smithi018.stdout:wait for servicemap items w/ changing names to refresh 2024-09-21T00:26:19.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:19 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:19 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:20.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:19 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:20.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'sleep 60' 2024-09-21T00:26:20.753 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:20 smithi018 ceph-mon[78227]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-09-21T00:26:20.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:20 smithi018 ceph-mon[80555]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-09-21T00:26:21.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:20 smithi031 ceph-mon[63863]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-09-21T00:26:22.011 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:21] "GET /metrics HTTP/1.1" 200 38433 "" "Prometheus/2.43.0" 2024-09-21T00:26:22.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:22 smithi018 ceph-mon[78227]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:22.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:22 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:22.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:22 smithi018 ceph-mon[80555]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:22.925 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:22 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:23.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:22 smithi031 ceph-mon[63863]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:23.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:22 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:25.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:24 smithi018 ceph-mon[78227]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:25.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:24 smithi018 ceph-mon[80555]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:25.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:24 smithi031 ceph-mon[63863]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:26 smithi018 ceph-mon[78227]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:26 smithi018 ceph-mon[80555]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:27.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:26 smithi031 ceph-mon[63863]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:29.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:28 smithi018 ceph-mon[78227]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:29.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:28 smithi018 ceph-mon[80555]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:29.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:28 smithi031 ceph-mon[63863]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:31.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:30 smithi018 ceph-mon[78227]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:31.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:30 smithi018 ceph-mon[80555]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:31.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:30 smithi031 ceph-mon[63863]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:32.204 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:31] "GET /metrics HTTP/1.1" 200 38433 "" "Prometheus/2.43.0" 2024-09-21T00:26:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:32 smithi018 ceph-mon[78227]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:33.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:32 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:32 smithi018 ceph-mon[80555]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:33.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:32 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:32 smithi031 ceph-mon[63863]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:33.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:32 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:34 smithi018 ceph-mon[78227]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:34 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:34 smithi018 ceph-mon[80555]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:34 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:35.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:34 smithi031 ceph-mon[63863]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:35.252 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:34 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:36 smithi018 ceph-mon[78227]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:36 smithi018 ceph-mon[80555]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:37.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:36 smithi031 ceph-mon[63863]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:39.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:38 smithi018 ceph-mon[78227]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:39.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:38 smithi018 ceph-mon[80555]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:39.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:38 smithi031 ceph-mon[63863]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:41.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:40 smithi018 ceph-mon[80555]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:41.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:40 smithi018 ceph-mon[78227]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:41.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:40 smithi031 ceph-mon[63863]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:42.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:41] "GET /metrics HTTP/1.1" 200 38432 "" "Prometheus/2.43.0" 2024-09-21T00:26:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:42 smithi018 ceph-mon[80555]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:43.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:42 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:42 smithi018 ceph-mon[78227]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:43.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:42 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:42 smithi031 ceph-mon[63863]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:43.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:42 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:45.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:44 smithi018 ceph-mon[80555]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:45.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:44 smithi018 ceph-mon[78227]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:45.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:44 smithi031 ceph-mon[63863]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:46 smithi018 ceph-mon[78227]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:46 smithi018 ceph-mon[80555]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:47.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:46 smithi031 ceph-mon[63863]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-21T00:26:49.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:48 smithi018 ceph-mon[78227]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:49.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:48 smithi018 ceph-mon[80555]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:49.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:48 smithi031 ceph-mon[63863]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:50.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:49 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:50.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:49 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:50.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:49 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:26:51.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:50 smithi018 ceph-mon[78227]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:51.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:50 smithi018 ceph-mon[80555]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:50 smithi031 ceph-mon[63863]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:26:52.216 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:26:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:26:51] "GET /metrics HTTP/1.1" 200 38432 "" "Prometheus/2.43.0" 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:53.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:53.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:52 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' 2024-09-21T00:26:53.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:52 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:26:55.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:55 smithi018 ceph-mon[78227]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:55.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:55 smithi018 ceph-mon[80555]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:55.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:55 smithi031 ceph-mon[63863]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:57.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:57 smithi018 ceph-mon[78227]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:57.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:57 smithi018 ceph-mon[80555]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:57.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:57 smithi031 ceph-mon[63863]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:59.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:26:59 smithi018 ceph-mon[78227]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:59.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:26:59 smithi018 ceph-mon[80555]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:26:59.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:26:59 smithi031 ceph-mon[63863]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:01.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:01 smithi018 ceph-mon[78227]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-09-21T00:27:01.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:01 smithi018 ceph-mon[80555]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-09-21T00:27:01.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:01 smithi031 ceph-mon[63863]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-09-21T00:27:02.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:01 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:01] "GET /metrics HTTP/1.1" 200 38432 "" "Prometheus/2.43.0" 2024-09-21T00:27:03.303 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:03 smithi031 ceph-mon[63863]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:27:03.303 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:03 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:03 smithi018 ceph-mon[78227]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:27:03.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:03 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:03 smithi018 ceph-mon[80555]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-21T00:27:03.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:03 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:05 smithi018 ceph-mon[78227]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:05.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:05 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:05 smithi018 ceph-mon[80555]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:05.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:05 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:05 smithi031 ceph-mon[63863]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:05.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:05 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:07.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:07 smithi018 ceph-mon[78227]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:07.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:07 smithi018 ceph-mon[80555]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:07.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:07 smithi031 ceph-mon[63863]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:09.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:09 smithi018 ceph-mon[78227]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:09.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:09 smithi018 ceph-mon[80555]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:09.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:09 smithi031 ceph-mon[63863]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:11.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:11 smithi018 ceph-mon[80555]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:11.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:11 smithi018 ceph-mon[78227]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:11.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:11 smithi031 ceph-mon[63863]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:12.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:11 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:11] "GET /metrics HTTP/1.1" 200 38433 "" "Prometheus/2.43.0" 2024-09-21T00:27:13.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:13 smithi018 ceph-mon[80555]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:13.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:13 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:13.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:13 smithi018 ceph-mon[78227]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:13.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:13 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:13.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:13 smithi031 ceph-mon[63863]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:13.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:13 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:15.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:15 smithi018 ceph-mon[78227]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:15.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:15 smithi018 ceph-mon[80555]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:15.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:15 smithi031 ceph-mon[63863]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:17.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:17 smithi018 ceph-mon[78227]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:17.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:17 smithi018 ceph-mon[80555]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:17.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:17 smithi031 ceph-mon[63863]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:19.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:19 smithi018 ceph-mon[78227]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:19.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:19 smithi018 ceph-mon[80555]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:19.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:19 smithi031 ceph-mon[63863]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:20.326 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:20 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:20.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:20 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:20.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:20 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:21.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:21 smithi018 ceph-mon[78227]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:21.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:21 smithi018 ceph-mon[80555]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:21.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:21 smithi031 ceph-mon[63863]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:22.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:21] "GET /metrics HTTP/1.1" 200 38429 "" "Prometheus/2.43.0" 2024-09-21T00:27:22.682 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ps' 2024-09-21T00:27:23.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:23 smithi018 ceph-mon[78227]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:23.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:23 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:23.246 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:23 smithi018 ceph-mon[80555]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:23.246 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:23 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:23 smithi031 ceph-mon[63863]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:23.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:23 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:24.929 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (24m) 2m ago 32m 23.9M - 0.25.0 c8568f914cd2 503cf3565fc2 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi031 *:3000 running (11m) 2m ago 32m 80.5M - 9.4.12 2bacad6d85d8 29b0278313a5 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:iscsi.foo.smithi018.wbwhhs smithi018 error 2m ago 31m 63.6M - 3.5 e1d6a67b021e b2ae45cb8f8e 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi031 *:8443,9283,8765 running (11m) 2m ago 37m 442M - 19.1.1-367-g9e2ac70b cc9251b744a3 e464d75c60d8 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (21m) 2m ago 38m 508M - 19.1.1-367-g9e2ac70b cc9251b744a3 e597087f2df9 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (9m) 2m ago 38m 62.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 8ccb4e6ec8c5 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi031 running (10m) 2m ago 37m 51.9M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 e2a83c256daa 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (9m) 2m ago 37m 49.5M 2048M 19.1.1-367-g9e2ac70b cc9251b744a3 32100aef662a 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (24m) 2m ago 33m 16.9M - 1.5.0 0da6a335fe13 b74742d0347a 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi031 *:9100 running (24m) 2m ago 33m 16.4M - 1.5.0 0da6a335fe13 2c775e5bd33d 2024-09-21T00:27:24.930 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (7m) 2m ago 36m 54.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 5a4be4b6542f 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (7m) 2m ago 36m 63.8M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 fceea84c4102 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (6m) 2m ago 36m 49.7M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 60c5194f7b2d 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (5m) 2m ago 35m 61.5M 2993M 19.1.1-367-g9e2ac70b cc9251b744a3 6575e2fcd7ba 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi031 running (4m) 2m ago 35m 52.4M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 2b0f22b23666 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi031 running (4m) 2m ago 34m 48.7M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 add05813eb99 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi031 running (3m) 2m ago 34m 48.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 330574b2ab7a 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi031 running (3m) 2m ago 34m 57.5M 3249M 19.1.1-367-g9e2ac70b cc9251b744a3 3a69d17a3c58 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi031 *:9095 running (11m) 2m ago 33m 71.4M - 2.43.0 a07b618ecd1d 93675685393e 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi018.kionxd smithi018 *:8000 running (2m) 2m ago 31m 76.8M - 19.1.1-367-g9e2ac70b cc9251b744a3 6a70459e4967 2024-09-21T00:27:24.931 INFO:teuthology.orchestra.run.smithi018.stdout:rgw.foo.smithi031.jzvksl smithi031 *:8000 running (2m) 2m ago 31m 78.9M - 19.1.1-367-g9e2ac70b cc9251b744a3 7f93ad9844ef 2024-09-21T00:27:25.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:25 smithi018 ceph-mon[78227]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:25.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:25 smithi018 ceph-mon[80555]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:25.508 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:25 smithi031 ceph-mon[63863]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:25.620 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions' 2024-09-21T00:27:26.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:26 smithi018 ceph-mon[78227]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:26.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:26 smithi018 ceph-mon[80555]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:26.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:26 smithi031 ceph-mon[63863]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:27.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:27 smithi018 ceph-mon[78227]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:27.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:27 smithi018 ceph-mon[80555]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:27.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:27 smithi031 ceph-mon[63863]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:27.756 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 3 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 8 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw": { 2024-09-21T00:27:27.757 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 2 2024-09-21T00:27:27.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T00:27:27.758 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T00:27:27.758 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)": 15 2024-09-21T00:27:27.758 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T00:27:27.758 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:27:28.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:28 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1279569657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:28.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:28 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1279569657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:28.409 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch upgrade status' 2024-09-21T00:27:28.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:28 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1279569657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:29.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:29 smithi018 ceph-mon[80555]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:29.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:29 smithi018 ceph-mon[78227]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:29.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:29 smithi031 ceph-mon[63863]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736", 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T00:27:30.395 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw", 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "osd", 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "15/21 daemons upgraded", 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed.", 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": true 2024-09-21T00:27:30.396 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:27:31.027 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph health detail' 2024-09-21T00:27:31.341 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:31 smithi018 ceph-mon[78227]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:31.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:31 smithi018 ceph-mon[80555]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:31.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:31 smithi031 ceph-mon[63863]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:32.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:32 smithi018 ceph-mon[78227]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:32.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:32 smithi018 ceph-mon[80555]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:32.175 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:31 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:31] "GET /metrics HTTP/1.1" 200 38429 "" "Prometheus/2.43.0" 2024-09-21T00:27:32.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:32 smithi031 ceph-mon[63863]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout: daemon iscsi.foo.smithi018.wbwhhs on smithi018 is in error state 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi018.wbwhhs on host smithi018 failed. 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout: Upgrade daemon: iscsi.foo.smithi018.wbwhhs: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi018.wbwhhs ... 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:Creating ceph-iscsi config... 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/iscsi-gateway.cfg 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:Write file: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/iscsi.foo.smithi018.wbwhhs/tcmu-runner-entrypoint.sh 2024-09-21T00:27:33.149 INFO:teuthology.orchestra.run.smithi018.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2dc1f9e122\x2d77aa\x2d11ef\x2dbaf5\x2defdc52797490.slice/ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl start ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout:Traceback (most recent call last): 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: return _run_code(code, main_globals, None, 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: exec(code, run_globals) 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5579, in 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 5567, in main 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-21T00:27:33.150 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1088, in deploy_daemon 2024-09-21T00:27:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/__main__.py", line 1224, in deploy_daemon_units 2024-09-21T00:27:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: File "/tmp/tmp3ebdx_ug.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-21T00:27:33.151 INFO:teuthology.orchestra.run.smithi018.stdout:RuntimeError: Failed command: systemctl start ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs: Job for ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service failed because the control process exited with error code. 2024-09-21T00:27:33.151 INFO:teuthology.orchestra.run.smithi018.stdout:See "systemctl status ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" and "journalctl -xeu ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@iscsi.foo.smithi018.wbwhhs.service" for details. 2024-09-21T00:27:33.402 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:33 smithi031 ceph-mon[63863]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:33.403 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:33 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:33.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:33 smithi018 ceph-mon[78227]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:33.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:33 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:33 smithi018 ceph-mon[80555]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:33.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:33 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:33.762 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T00:27:34.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:34 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/3363465565' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:27:34.401 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:34 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/3363465565' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:27:34.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:34 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/3363465565' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T00:27:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:35 smithi018 ceph-mon[78227]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:35.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:35 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:35 smithi018 ceph-mon[80555]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:35.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:35 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:35 smithi031 ceph-mon[63863]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:35.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:35 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:35.917 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T00:27:36.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:36 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1683391130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:36.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:36 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1683391130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:36.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T00:27:36.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:36 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1683391130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:37.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:37 smithi018 ceph-mon[78227]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:37.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:37 smithi018 ceph-mon[80555]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:37.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:37 smithi031 ceph-mon[63863]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:38.590 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.1.1-367-g9e2ac70b (9e2ac70bb7aea67d34f0d50002344abbe7803736) squid (stable)" 2024-09-21T00:27:39.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:39 smithi018 ceph-mon[80555]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:39.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:39 smithi018 ceph-mon[80555]: from='client.? 172.21.15.18:0/1278037253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:39.169 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:39 smithi018 ceph-mon[78227]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:39.169 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:39 smithi018 ceph-mon[78227]: from='client.? 172.21.15.18:0/1278037253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:39.193 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -e sha1=9e2ac70bb7aea67d34f0d50002344abbe7803736 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-09-21T00:27:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:39 smithi031 ceph-mon[63863]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:39.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:39 smithi031 ceph-mon[63863]: from='client.? 172.21.15.18:0/1278037253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T00:27:41.180 INFO:teuthology.orchestra.run.smithi018.stdout:osd 8 2m ago - 2024-09-21T00:27:41.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:41 smithi018 ceph-mon[78227]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:41.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:41 smithi018 ceph-mon[80555]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:41.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:41 smithi031 ceph-mon[63863]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:41.795 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T00:27:41.835 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T00:27:41.835 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- bash -c 'ceph orch upgrade ls' 2024-09-21T00:27:42.126 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:41 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:41] "GET /metrics HTTP/1.1" 200 38431 "" "Prometheus/2.43.0" 2024-09-21T00:27:42.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:42 smithi018 ceph-mon[80555]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:42.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:42 smithi018 ceph-mon[78227]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:42.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:42 smithi031 ceph-mon[63863]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:43 smithi018 ceph-mon[78227]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:43.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:43 smithi018 ceph-mon[78227]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:43 smithi018 ceph-mon[80555]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:43.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:43 smithi018 ceph-mon[80555]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:43 smithi031 ceph-mon[63863]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:43.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:43 smithi031 ceph-mon[63863]: from='client.54748 -' entity='client.iscsi.foo.smithi018.wbwhhs' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "image": "quay.io/ceph/ceph", 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "registry": "quay.io", 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "bare_image": "ceph/ceph", 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "versions": [ 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "19.1.1", 2024-09-21T00:27:44.146 INFO:teuthology.orchestra.run.smithi018.stdout: "19.1.0" 2024-09-21T00:27:44.147 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-21T00:27:44.147 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T00:27:44.788 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-09-21T00:27:45.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:45 smithi018 ceph-mon[78227]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:45.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:45 smithi018 ceph-mon[78227]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:45.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:45 smithi018 ceph-mon[80555]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:45.426 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:45 smithi018 ceph-mon[80555]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:45 smithi031 ceph-mon[63863]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:45.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:45 smithi031 ceph-mon[63863]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:47.174 INFO:teuthology.orchestra.run.smithi018.stdout: "16.2.0", 2024-09-21T00:27:47.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:47 smithi018 ceph-mon[78227]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:47.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:47 smithi018 ceph-mon[80555]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:47.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:47 smithi031 ceph-mon[63863]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-21T00:27:47.835 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-09-21T00:27:48.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:48 smithi018 ceph-mon[78227]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:48.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:48 smithi018 ceph-mon[80555]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:48.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:48 smithi031 ceph-mon[63863]: from='client.44940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:49.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:49 smithi018 ceph-mon[78227]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:49.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:49 smithi018 ceph-mon[80555]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:49.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:49 smithi031 ceph-mon[63863]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:50.238 INFO:teuthology.orchestra.run.smithi018.stdout: "v16.2.2", 2024-09-21T00:27:50.238 INFO:teuthology.orchestra.run.smithi018.stdout: "v16.2.2-20210505", 2024-09-21T00:27:50.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:50 smithi018 ceph-mon[78227]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:50.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:50 smithi018 ceph-mon[80555]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:50.501 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:50 smithi031 ceph-mon[63863]: from='mgr.25579 172.21.15.18:0/2666943364' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T00:27:50.903 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T00:27:50.912 INFO:tasks.cephadm:Teardown begin 2024-09-21T00:27:50.912 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T00:27:50.941 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T00:27:50.975 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T00:27:50.976 DEBUG:teuthology.orchestra.run.smithi018:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T00:27:50.997 DEBUG:teuthology.orchestra.run.smithi031:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T00:27:51.034 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T00:27:51.035 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T00:27:51.035 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a 2024-09-21T00:27:51.152 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 systemd[1]: Stopping Ceph mon.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:51 smithi031 ceph-mon[63863]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:51.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:51 smithi031 ceph-mon[63863]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:51.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-mon[80555]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-mon[80555]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE Bus STOPPING 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-mon[78227]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-mon[78227]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[78223]: 2024-09-21T00:27:51.347+0000 7fa40cd01640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:27:51.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a[78223]: 2024-09-21T00:27:51.347+0000 7fa40cd01640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T00:27:51.426 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 podman[121776]: 2024-09-21 00:27:51.368785827 +0000 UTC m=+0.134412383 container died 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T00:27:51.740 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:27:51.740 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE Bus STOPPED 2024-09-21T00:27:51.740 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE Bus STARTING 2024-09-21T00:27:51.740 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE Serving on http://:::9283 2024-09-21T00:27:51.740 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:51] ENGINE Bus STARTED 2024-09-21T00:27:51.991 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:51 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: ::ffff:172.21.15.31 - - [21/Sep/2024:00:27:51] "GET /metrics HTTP/1.1" 200 38430 "" "Prometheus/2.43.0" 2024-09-21T00:27:51.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 podman[121776]: 2024-09-21 00:27:51.739942262 +0000 UTC m=+0.505568854 container cleanup 8ccb4e6ec8c5dd4c708eb1b2a490d46e5c691ca9c44379be8e91cdc72e352890 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T00:27:51.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 21 00:27:51 smithi018 bash[121776]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-a 2024-09-21T00:27:52.143 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.a.service' 2024-09-21T00:27:52.193 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:27:52.193 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T00:27:52.193 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T00:27:52.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.c 2024-09-21T00:27:52.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:52 smithi018 systemd[1]: Stopping Ceph mon.c for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:52.830 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:52] ENGINE Bus STOPPING 2024-09-21T00:27:52.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c[80551]: 2024-09-21T00:27:52.564+0000 7f8e1b99a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:27:52.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c[80551]: 2024-09-21T00:27:52.564+0000 7f8e1b99a640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T00:27:53.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:52 smithi018 podman[121910]: 2024-09-21 00:27:52.82927451 +0000 UTC m=+0.399547201 container died 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T00:27:53.129 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T00:27:53.130 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:52] ENGINE Bus STOPPED 2024-09-21T00:27:53.130 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:52 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:52] ENGINE Bus STARTING 2024-09-21T00:27:53.130 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:53] ENGINE Serving on http://:::9283 2024-09-21T00:27:53.130 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:53 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: [21/Sep/2024:00:27:53] ENGINE Bus STARTED 2024-09-21T00:27:53.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:53 smithi018 podman[121910]: 2024-09-21 00:27:53.219530286 +0000 UTC m=+0.789802967 container cleanup 32100aef662a39a4bffdb3c70b5f880c4f6c7e7fed6397b5c08ea6fef7f1c022 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-9e2ac70) 2024-09-21T00:27:53.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 21 00:27:53 smithi018 bash[121910]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-c 2024-09-21T00:27:53.613 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.c.service' 2024-09-21T00:27:53.652 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:27:53.652 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T00:27:53.652 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T00:27:53.652 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.b 2024-09-21T00:27:53.962 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:53 smithi031 systemd[1]: Stopping Ceph mon.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b[63859]: 2024-09-21T00:27:53.959+0000 7feb51cff640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:27:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b[63859]: 2024-09-21T00:27:53.959+0000 7feb51cff640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T00:27:54.251 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:54 smithi031 podman[96913]: 2024-09-21 00:27:54.185206461 +0000 UTC m=+0.333119663 container died e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T00:27:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:54 smithi031 podman[96913]: 2024-09-21 00:27:54.76057292 +0000 UTC m=+0.908486114 container cleanup e2a83c256daa1188e643039a12bce19adc571b2ebd6d4d5f8aa8d968b040db9c (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux ) 2024-09-21T00:27:55.001 INFO:journalctl@ceph.mon.b.smithi031.stdout:Sep 21 00:27:54 smithi031 bash[96913]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mon-b 2024-09-21T00:27:55.204 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mon.b.service' 2024-09-21T00:27:55.336 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:27:55.337 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T00:27:55.337 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T00:27:55.337 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y 2024-09-21T00:27:55.650 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:55 smithi018 systemd[1]: Stopping Ceph mgr.y for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:55.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:27:55.648+0000 7fd0aea39640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:27:55.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:55 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y[61427]: 2024-09-21T00:27:55.648+0000 7fd0aea39640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-21T00:27:55.925 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:55 smithi018 podman[122051]: 2024-09-21 00:27:55.690374152 +0000 UTC m=+0.153529476 container died e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:27:56.309 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:56 smithi018 podman[122051]: 2024-09-21 00:27:56.065717362 +0000 UTC m=+0.528872685 container cleanup e597087f2df922bb3f2b7232d1e5d0a1c46ec8831aff80526e9b004084839db0 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux ) 2024-09-21T00:27:56.309 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 21 00:27:56 smithi018 bash[122051]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-y 2024-09-21T00:27:56.463 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.y.service' 2024-09-21T00:27:56.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:27:56.499 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T00:27:56.499 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T00:27:56.499 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x 2024-09-21T00:27:56.847 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:27:56 smithi031 systemd[1]: Stopping Ceph mgr.x for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:57.146 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:27:56 smithi031 podman[97033]: 2024-09-21 00:27:56.846365564 +0000 UTC m=+0.137545281 container died e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-9e2ac70, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T00:27:57.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:27:57 smithi031 podman[97033]: 2024-09-21 00:27:57.34885073 +0000 UTC m=+0.640030481 container cleanup e464d75c60d8869ba0c2457944db9af00e4258ba80dc6e5c0571bead9c05a7b4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:27:57.501 INFO:journalctl@ceph.mgr.x.smithi031.stdout:Sep 21 00:27:57 smithi031 bash[97033]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-mgr-x 2024-09-21T00:27:57.736 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@mgr.x.service' 2024-09-21T00:27:57.771 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:27:57.771 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T00:27:57.771 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T00:27:57.771 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.0 2024-09-21T00:27:58.174 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:27:57 smithi018 systemd[1]: Stopping Ceph osd.0 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:27:58.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:27:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:27:58.044+0000 7feb84920640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:27:58.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:27:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:27:58.044+0000 7feb84920640 -1 osd.0 167 *** Got signal Terminated *** 2024-09-21T00:27:58.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:27:58 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0[87911]: 2024-09-21T00:27:58.044+0000 7feb84920640 -1 osd.0 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:03.663 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:03 smithi018 podman[122174]: 2024-09-21 00:28:03.393877987 +0000 UTC m=+5.427881774 container died 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T00:28:03.924 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:03 smithi018 podman[122174]: 2024-09-21 00:28:03.761903767 +0000 UTC m=+5.795907552 container cleanup 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T00:28:03.924 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:03 smithi018 bash[122174]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0 2024-09-21T00:28:04.391 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122189]: 2024-09-21 00:28:04.038290192 +0000 UTC m=+0.638968657 container remove 5a4be4b6542f2641548d3ab13591552df85f381b116729af421bb3433f5d9ce4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T00:28:04.392 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:04.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:04.392 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:04.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:04.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.294263293 +0000 UTC m=+0.016995063 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:04.675 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.439206118 +0000 UTC m=+0.161937875 container create c6795e213a9f06fef1f40c64082e6e68cb9be4d4ed59bec2a647fa8e85a8f4cb (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T00:28:05.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.675489438 +0000 UTC m=+0.398221206 container init c6795e213a9f06fef1f40c64082e6e68cb9be4d4ed59bec2a647fa8e85a8f4cb (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:28:05.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.679517265 +0000 UTC m=+0.402249037 container start c6795e213a9f06fef1f40c64082e6e68cb9be4d4ed59bec2a647fa8e85a8f4cb (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:05.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.741519136 +0000 UTC m=+0.464250912 container attach c6795e213a9f06fef1f40c64082e6e68cb9be4d4ed59bec2a647fa8e85a8f4cb (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T00:28:05.175 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 00:28:04 smithi018 podman[122275]: 2024-09-21 00:28:04.869695059 +0000 UTC m=+0.592426833 container died c6795e213a9f06fef1f40c64082e6e68cb9be4d4ed59bec2a647fa8e85a8f4cb (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70) 2024-09-21T00:28:05.600 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.0.service' 2024-09-21T00:28:05.645 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:05.645 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T00:28:05.645 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T00:28:05.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.1 2024-09-21T00:28:05.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:05 smithi018 systemd[1]: Stopping Ceph osd.1 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:06.152 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:28:06.024+0000 7faa728ff640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:06.152 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:28:06.024+0000 7faa728ff640 -1 osd.1 167 *** Got signal Terminated *** 2024-09-21T00:28:06.152 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:06 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1[92761]: 2024-09-21T00:28:06.024+0000 7faa728ff640 -1 osd.1 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:11.751 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:11 smithi018 podman[122380]: 2024-09-21 00:28:11.435726092 +0000 UTC m=+5.596391237 container died fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T00:28:12.050 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:11 smithi018 podman[122380]: 2024-09-21 00:28:11.839859692 +0000 UTC m=+6.000524820 container cleanup fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T00:28:12.050 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:11 smithi018 bash[122380]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1 2024-09-21T00:28:12.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122409]: 2024-09-21 00:28:12.049474112 +0000 UTC m=+0.609354906 container remove fceea84c41021565444e16584a972ce2cb05a267be85aec4bcd022a14e84c868 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-9e2ac70, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T00:28:12.653 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.391824027 +0000 UTC m=+0.112655645 container create b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T00:28:12.653 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.295810265 +0000 UTC m=+0.016641885 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:12.654 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.577717468 +0000 UTC m=+0.298549094 container init b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T00:28:12.654 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.581216071 +0000 UTC m=+0.302047696 container start b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T00:28:12.654 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.652722335 +0000 UTC m=+0.373553976 container attach b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T00:28:12.925 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:12 smithi018 podman[122500]: 2024-09-21 00:28:12.775845036 +0000 UTC m=+0.496676666 container died b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T00:28:13.313 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 00:28:13 smithi018 podman[122523]: 2024-09-21 00:28:13.297760866 +0000 UTC m=+0.515698483 container remove b5b30a35c3f435ae5213cabccd8be2ea84e66f2f6534d9488245f8e05c8d14fd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T00:28:13.319 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.1.service' 2024-09-21T00:28:13.362 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:13.363 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T00:28:13.363 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T00:28:13.363 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2 2024-09-21T00:28:13.674 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:13 smithi018 systemd[1]: Stopping Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:14.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:28:13.709+0000 7f291af4d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:14.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:28:13.709+0000 7f291af4d640 -1 osd.2 167 *** Got signal Terminated *** 2024-09-21T00:28:14.175 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:13 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2[99243]: 2024-09-21T00:28:13.709+0000 7f291af4d640 -1 osd.2 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:14.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:14.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:14.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:14.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:14.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:14 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:14.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:19.361 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:19 smithi018 podman[122599]: 2024-09-21 00:28:19.07398607 +0000 UTC m=+5.480127796 container died 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T00:28:19.669 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:19 smithi018 podman[122599]: 2024-09-21 00:28:19.450191683 +0000 UTC m=+5.856333407 container cleanup 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T00:28:19.669 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:19 smithi018 bash[122599]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2 2024-09-21T00:28:19.925 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:19 smithi018 podman[122614]: 2024-09-21 00:28:19.668520435 +0000 UTC m=+0.588670925 container remove 60c5194f7b2d860741901fb9e528ddce79c3c8257b500baf1d555ded3631849a (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T00:28:20.331 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:19.903377774 +0000 UTC m=+0.016631827 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:20.331 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:20.019293325 +0000 UTC m=+0.132547358 container create e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux ) 2024-09-21T00:28:20.331 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:20.246767017 +0000 UTC m=+0.360021050 container init e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:28:20.331 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:20.250307955 +0000 UTC m=+0.363561994 container start e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T00:28:20.332 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:20.329940563 +0000 UTC m=+0.443194597 container attach e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T00:28:20.675 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:20 smithi018 podman[122703]: 2024-09-21 00:28:20.445065544 +0000 UTC m=+0.558319577 container died e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T00:28:21.027 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:21 smithi018 podman[122726]: 2024-09-21 00:28:21.008711742 +0000 UTC m=+0.555025716 container remove e14df98a1f401ebacab42f37ad1238820f1f40cb0be21f397910904fa3687176 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:21.027 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:21 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T00:28:21.027 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:21 smithi018 systemd[1]: Stopped Ceph osd.2 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:28:21.027 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 00:28:21 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service: Consumed 4.981s CPU time. 2024-09-21T00:28:21.029 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.2.service' 2024-09-21T00:28:21.073 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:21.074 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T00:28:21.074 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T00:28:21.074 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3 2024-09-21T00:28:21.424 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:21 smithi018 systemd[1]: Stopping Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:21.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:28:21.428+0000 7f161d68e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:21.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:28:21.428+0000 7f161d68e640 -1 osd.3 167 *** Got signal Terminated *** 2024-09-21T00:28:21.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:21 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:28:21.428+0000 7f161d68e640 -1 osd.3 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:24.251 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:24 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:24.140+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:24.252 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:23 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:23.973+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:24.670 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:24 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:24.275Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:24.671 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:24 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:24.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:24.671 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:24 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:24.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:24.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:24 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:24.478+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:24.752 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:24 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:24.699+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:24.925 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:24 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:28:24.669+0000 7f1615c9c640 -1 osd.3 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:00.376597+0000 front 2024-09-21T00:28:00.376642+0000 (oldest deadline 2024-09-21T00:28:23.777219+0000) 2024-09-21T00:28:25.486 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:25 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:25.161+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:25.486 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:25 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:25.020+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:25.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:25 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:25.484+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:25.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:25 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:25.665+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:26.175 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:25 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3[105761]: 2024-09-21T00:28:25.684+0000 7f1615c9c640 -1 osd.3 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:00.376597+0000 front 2024-09-21T00:28:00.376642+0000 (oldest deadline 2024-09-21T00:28:23.777219+0000) 2024-09-21T00:28:26.477 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:26.135+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:26.477 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:26.060+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:26.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:26.475+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:26.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:26 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:26.667+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:27.122 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:26 smithi018 podman[122806]: 2024-09-21 00:28:26.762943517 +0000 UTC m=+5.449401105 container died 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T00:28:27.422 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:27 smithi018 podman[122806]: 2024-09-21 00:28:27.195105922 +0000 UTC m=+5.881563505 container cleanup 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T00:28:27.422 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:27 smithi018 bash[122806]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3 2024-09-21T00:28:27.443 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:27.154+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:27.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:27.048+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:27.674 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:27 smithi018 podman[122831]: 2024-09-21 00:28:27.421559746 +0000 UTC m=+0.649924166 container remove 6575e2fcd7ba00f925f6f9014ba77c765df5f88259697f6bada24f1ce7e988b7 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T00:28:27.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:27.691+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:27.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:27 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:27.441+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:28.116 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:27 smithi018 podman[122925]: 2024-09-21 00:28:27.735488817 +0000 UTC m=+0.016405640 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:28.117 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:27 smithi018 podman[122925]: 2024-09-21 00:28:27.847207281 +0000 UTC m=+0.128124100 container create 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T00:28:28.117 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 podman[122925]: 2024-09-21 00:28:28.041890651 +0000 UTC m=+0.322807481 container init 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-9e2ac70, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:28:28.117 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 podman[122925]: 2024-09-21 00:28:28.04584736 +0000 UTC m=+0.326764181 container start 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70) 2024-09-21T00:28:28.117 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 podman[122925]: 2024-09-21 00:28:28.116229985 +0000 UTC m=+0.397146824 container attach 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:28.425 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 podman[122925]: 2024-09-21 00:28:28.237447789 +0000 UTC m=+0.518364610 container died 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD) 2024-09-21T00:28:28.425 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:28.183+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:28.425 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:28.086+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:28.698 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:28.424+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:28.806 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service' 2024-09-21T00:28:28.831 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 podman[122948]: 2024-09-21 00:28:28.786468298 +0000 UTC m=+0.542919552 container remove 601f926395f30e4eef7ba94a7d46aca139cea2358a1dfbe66b033864b39297a4 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:28:28.831 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T00:28:28.831 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 systemd[1]: Stopped Ceph osd.3 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:28:28.831 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 00:28:28 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.3.service: Consumed 6.403s CPU time. 2024-09-21T00:28:28.847 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:28.847 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T00:28:28.847 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T00:28:28.848 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.4 2024-09-21T00:28:29.001 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:28 smithi031 systemd[1]: Stopping Ceph osd.4 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:29.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:28 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:28.697+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:29.430 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:29.066+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:29.430 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:29.158+0000 7faf0934f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:29.430 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:29.158+0000 7faf0934f640 -1 osd.4 167 *** Got signal Terminated *** 2024-09-21T00:28:29.430 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:29.158+0000 7faf0934f640 -1 osd.4 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:29.431 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:29.191+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:29.698 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:29.429+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:30.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:29 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:29.697+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:30.415 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:30 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:30.226+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:30.415 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:30 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:30.044+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:30.710 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:30 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:30.414+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:31.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:30 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:30.709+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:31.411 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:31.205+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:31.411 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:31.089+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:31.726 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:31.409+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:32.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:31.725+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:32.448 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:32.178+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:32.448 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:32.058+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:32.716 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:32.447+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:33.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:32.715+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:33.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:32 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:32.715+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:33.433 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:33.197+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:28:02.922508+0000 front 2024-09-21T00:28:02.922283+0000 (oldest deadline 2024-09-21T00:28:23.422297+0000) 2024-09-21T00:28:33.433 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4[74177]: 2024-09-21T00:28:33.197+0000 7faf05965640 -1 osd.4 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.924337+0000 front 2024-09-21T00:28:10.924427+0000 (oldest deadline 2024-09-21T00:28:32.624103+0000) 2024-09-21T00:28:33.434 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:33.093+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:33.434 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:33.093+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:33.707 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:33.432+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:33.707 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:33.432+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:34.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:33.705+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:34.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:33 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:33.705+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:34.432 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:34.137+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:34.432 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:34.137+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:34.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:34.276Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:34.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:34.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:34.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:34 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:34.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:34.705 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:34 smithi031 podman[97159]: 2024-09-21 00:28:34.513638164 +0000 UTC m=+5.465931220 container died 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:34.705 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:34.431+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:34.705 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:34.431+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:35.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:34.703+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:35.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:34 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:34.703+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:35.384 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97159]: 2024-09-21 00:28:35.092690638 +0000 UTC m=+6.044983722 container cleanup 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-9e2ac70, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:35.385 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 bash[97159]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4 2024-09-21T00:28:35.385 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97173]: 2024-09-21 00:28:35.335960387 +0000 UTC m=+0.818171747 container remove 2b0f22b23666bd5d74b80a28b28c6b400b20319d38913523151fbf76a2fcc91b (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T00:28:35.385 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:35.101+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:35.385 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:35.101+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:35.670 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:35.383+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:35.670 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:35.383+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:35.948 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97263]: 2024-09-21 00:28:35.572716349 +0000 UTC m=+0.016794392 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:35.948 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97263]: 2024-09-21 00:28:35.669971824 +0000 UTC m=+0.114049873 container create 16297456ce8dc3e65860cacb00a2d0ac059911538ea6a41e3ca98b373f4665dd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:35.948 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97263]: 2024-09-21 00:28:35.864244517 +0000 UTC m=+0.308322569 container init 16297456ce8dc3e65860cacb00a2d0ac059911538ea6a41e3ca98b373f4665dd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:35.948 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97263]: 2024-09-21 00:28:35.867579057 +0000 UTC m=+0.311657100 container start 16297456ce8dc3e65860cacb00a2d0ac059911538ea6a41e3ca98b373f4665dd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T00:28:35.949 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:35.701+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:35.949 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:35 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:35.701+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:36.251 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:35 smithi031 podman[97263]: 2024-09-21 00:28:35.947408216 +0000 UTC m=+0.391486276 container attach 16297456ce8dc3e65860cacb00a2d0ac059911538ea6a41e3ca98b373f4665dd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T00:28:36.252 INFO:journalctl@ceph.osd.4.smithi031.stdout:Sep 21 00:28:36 smithi031 podman[97263]: 2024-09-21 00:28:36.069993312 +0000 UTC m=+0.514071361 container died 16297456ce8dc3e65860cacb00a2d0ac059911538ea6a41e3ca98b373f4665dd (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T00:28:36.252 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:36.106+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:36.252 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:36.106+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:36.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:36.399+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:36.502 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:36.399+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:36.768 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:36.661+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:36.769 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:36 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:36.661+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:36.785 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.4.service' 2024-09-21T00:28:36.829 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:36.829 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T00:28:36.830 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T00:28:36.830 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5 2024-09-21T00:28:37.135 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:36 smithi031 systemd[1]: Stopping Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:37.432 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:37.179+0000 7fb6c44c1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:37.432 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:37.179+0000 7fb6c44c1640 -1 osd.5 167 *** Got signal Terminated *** 2024-09-21T00:28:37.432 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:37.179+0000 7fb6c44c1640 -1 osd.5 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:37.432 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:37.133+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:37.432 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:37.133+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:37.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:37.431+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:37.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:37.431+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:37.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:37.661+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:37.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:37 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:37.661+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:38.442 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:38.171+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:38.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:38.171+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:38.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:38.441+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:38.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:38.441+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:38.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:38.633+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:38.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:38 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:38.633+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:39.412 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:39.123+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:39.412 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:39.123+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:39.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:39.410+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:39.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:39.410+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:39.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:39.612+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:39.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:39 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:39.612+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:40.393 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:40.079+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:40.393 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:40.079+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:40.393 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:40.079+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:40.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:40.391+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:40.751 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:40.391+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:40.752 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:40.578+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:40.752 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:40.578+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:40.752 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:40 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:40.578+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:41.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:41.359+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.785735+0000 front 2024-09-21T00:27:58.785800+0000 (oldest deadline 2024-09-21T00:28:23.485573+0000) 2024-09-21T00:28:41.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:41.359+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:08.687337+0000 front 2024-09-21T00:28:08.687185+0000 (oldest deadline 2024-09-21T00:28:32.787041+0000) 2024-09-21T00:28:41.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5[78807]: 2024-09-21T00:28:41.359+0000 7fb6c0ad7640 -1 osd.5 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:15.687704+0000 front 2024-09-21T00:28:15.687850+0000 (oldest deadline 2024-09-21T00:28:40.987916+0000) 2024-09-21T00:28:41.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:41.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:41.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:41.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:41.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:41.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:42.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:41.532+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:42.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:41.532+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:42.002 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:41 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:41.532+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:42.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:42.142+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:42.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:42.142+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:42.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:42.142+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:42.856 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:42 smithi031 podman[97366]: 2024-09-21 00:28:42.572054087 +0000 UTC m=+5.506441828 container died add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T00:28:42.856 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:42.560+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:42.857 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:42.560+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:42.857 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:42 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:42.560+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:43.185 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97366]: 2024-09-21 00:28:43.046141004 +0000 UTC m=+5.980528744 container cleanup add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-9e2ac70, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T00:28:43.185 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 bash[97366]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5 2024-09-21T00:28:43.501 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97381]: 2024-09-21 00:28:43.305798865 +0000 UTC m=+0.731763522 container remove add05813eb99d57e940fa6650e63ef4068470be4523068203e57b1cf93eff285 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T00:28:43.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:43.183+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:43.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:43.183+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:43.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:43.183+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:43.877 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97475]: 2024-09-21 00:28:43.532703248 +0000 UTC m=+0.016599239 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:43.877 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97475]: 2024-09-21 00:28:43.631625903 +0000 UTC m=+0.115521884 container create a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T00:28:43.877 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:43.605+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:43.877 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:43.605+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:43.877 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:43 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:43.605+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:44.158 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97475]: 2024-09-21 00:28:43.876370971 +0000 UTC m=+0.360266964 container init a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:44.158 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97475]: 2024-09-21 00:28:43.880198386 +0000 UTC m=+0.364094370 container start a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T00:28:44.158 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:43 smithi031 podman[97475]: 2024-09-21 00:28:43.959378843 +0000 UTC m=+0.443274835 container attach a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T00:28:44.158 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:44 smithi031 podman[97475]: 2024-09-21 00:28:44.075315971 +0000 UTC m=+0.559211962 container died a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T00:28:44.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:44.157+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:44.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:44.157+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:44.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:44.157+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:44.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:44.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:44.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:44.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:44.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:44 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:44.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:44.797 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service' 2024-09-21T00:28:44.824 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:44 smithi031 podman[97498]: 2024-09-21 00:28:44.771894469 +0000 UTC m=+0.690430106 container remove a59ef339dd6b3fecca1376b0bcf4a0fa604ebeabbbd37134c41d763795e28ce1 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T00:28:44.824 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:44 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T00:28:44.825 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:44 smithi031 systemd[1]: Stopped Ceph osd.5 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:28:44.825 INFO:journalctl@ceph.osd.5.smithi031.stdout:Sep 21 00:28:44 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.5.service: Consumed 4.995s CPU time. 2024-09-21T00:28:44.825 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:44.566+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:44.825 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:44.566+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:44.825 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:44 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:44.566+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:44.842 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:44.842 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T00:28:44.842 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T00:28:44.842 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6 2024-09-21T00:28:45.110 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:44 smithi031 systemd[1]: Stopping Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:45.501 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.200+0000 7f905108b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:45.501 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.200+0000 7f905108b640 -1 osd.6 167 *** Got signal Terminated *** 2024-09-21T00:28:45.501 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.200+0000 7f905108b640 -1 osd.6 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:45.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:45.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:45.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:45.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:45.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:45.109+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:46.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.563+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:46.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.563+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:46.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:45 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:45.563+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:46.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:46.140+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:46.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:46.140+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:46.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:46.140+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:47.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:46.592+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:47.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:46.592+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:47.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:46 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:46.592+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:47.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:47.138+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:47.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:47.138+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:47.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:47.138+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:47.501 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:28:47.011Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.18:8765: connect: connection refused" 2024-09-21T00:28:47.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:28:47.014Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.18:8765: connect: connection refused" 2024-09-21T00:28:47.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:28:47.014Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.18:8765: connect: connection refused" 2024-09-21T00:28:47.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:28:47.014Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.18:8765: connect: connection refused" 2024-09-21T00:28:47.502 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:28:47.015Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.18:8765: connect: connection refused" 2024-09-21T00:28:48.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:47.580+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:48.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:47.580+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:48.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:47.580+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:48.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:47 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:47.580+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:24.852611+0000 front 2024-09-21T00:28:24.852556+0000 (oldest deadline 2024-09-21T00:28:47.152544+0000) 2024-09-21T00:28:48.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:48.169+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:48.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:48.169+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:48.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:48.169+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:49.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:48.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:49.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:48.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:49.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:48.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:49.002 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:48 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:48.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:24.852611+0000 front 2024-09-21T00:28:24.852556+0000 (oldest deadline 2024-09-21T00:28:47.152544+0000) 2024-09-21T00:28:49.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:49.171+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:49.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:49.171+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:49.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:49.171+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:50.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:49.549+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.748296+0000 front 2024-09-21T00:27:58.748187+0000 (oldest deadline 2024-09-21T00:28:24.048128+0000) 2024-09-21T00:28:50.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:49.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:10.250662+0000 front 2024-09-21T00:28:10.250702+0000 (oldest deadline 2024-09-21T00:28:31.950682+0000) 2024-09-21T00:28:50.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:49.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:14.851661+0000 front 2024-09-21T00:28:14.851687+0000 (oldest deadline 2024-09-21T00:28:40.151554+0000) 2024-09-21T00:28:50.001 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:49 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6[83733]: 2024-09-21T00:28:49.550+0000 7f904d6a1640 -1 osd.6 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:24.852611+0000 front 2024-09-21T00:28:24.852556+0000 (oldest deadline 2024-09-21T00:28:47.152544+0000) 2024-09-21T00:28:50.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:50.156+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:50.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:50.156+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:50.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:50 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:50.156+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:50.900 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:50 smithi031 podman[97577]: 2024-09-21 00:28:50.510955409 +0000 UTC m=+5.434370758 container died 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T00:28:50.901 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:50 smithi031 podman[97577]: 2024-09-21 00:28:50.900218362 +0000 UTC m=+5.823633742 container cleanup 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T00:28:51.208 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:50 smithi031 bash[97577]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6 2024-09-21T00:28:51.208 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97592]: 2024-09-21 00:28:51.126541838 +0000 UTC m=+0.611918864 container remove 330574b2ab7a459a3a8270c5313c729872c6094d4c8f0ade96106726aefe8ca8 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T00:28:51.466 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:51.206+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:51.466 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:51.206+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:51.466 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:51.206+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:51.466 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:51 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:51.206+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:51.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.368264813 +0000 UTC m=+0.016457007 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:51.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.485561643 +0000 UTC m=+0.133753828 container create bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70) 2024-09-21T00:28:51.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.704393888 +0000 UTC m=+0.352586070 container init bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T00:28:51.751 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.708044019 +0000 UTC m=+0.356236210 container start bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-9e2ac70, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T00:28:52.192 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.77947576 +0000 UTC m=+0.427667942 container attach bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T00:28:52.192 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:51 smithi031 podman[97682]: 2024-09-21 00:28:51.904989686 +0000 UTC m=+0.553181880 container died bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T00:28:52.442 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:52 smithi031 podman[97705]: 2024-09-21 00:28:52.424874589 +0000 UTC m=+0.513681986 container remove bc10f7b1366a92399e5120653e30a047ea3f94489cf9af2396a2b87bba1aa451 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-9e2ac70, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:52 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:52 smithi031 systemd[1]: Stopped Ceph osd.6 for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.6.smithi031.stdout:Sep 21 00:28:52 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service: Consumed 4.261s CPU time. 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.215+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.215+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.215+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:52.443 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.215+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:52.444 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.6.service' 2024-09-21T00:28:52.484 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:28:52.485 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T00:28:52.485 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T00:28:52.485 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.7 2024-09-21T00:28:52.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 systemd[1]: Stopping Ceph osd.7 for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:28:53.251 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.877+0000 7ff2207fc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T00:28:53.251 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.877+0000 7ff2207fc640 -1 osd.7 167 *** Got signal Terminated *** 2024-09-21T00:28:53.251 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:52 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:52.877+0000 7ff2207fc640 -1 osd.7 167 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T00:28:53.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:53.262+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:53.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:53.262+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:53.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:53.262+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:53.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:53 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:53.262+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:54.585 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:54.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:54.586 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:54.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:28:54.586 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:28:54 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:28:54.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:28:54.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:54.252+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:54.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:54.252+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:54.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:54.252+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:54.751 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:54 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:54.252+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:55.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:55.233+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:55.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:55.233+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:55.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:55.233+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:55.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:55 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:55.233+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:56.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:56.224+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:56.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:56.224+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:56.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:56.224+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:56.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:56 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:56.224+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:57.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:57.177+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T00:27:58.653349+0000 front 2024-09-21T00:27:58.653435+0000 (oldest deadline 2024-09-21T00:28:23.953250+0000) 2024-09-21T00:28:57.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:57.177+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T00:28:07.855271+0000 front 2024-09-21T00:28:07.855256+0000 (oldest deadline 2024-09-21T00:28:32.555191+0000) 2024-09-21T00:28:57.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:57.177+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T00:28:17.756490+0000 front 2024-09-21T00:28:17.756591+0000 (oldest deadline 2024-09-21T00:28:39.956838+0000) 2024-09-21T00:28:57.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:57.177+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T00:28:25.257811+0000 front 2024-09-21T00:28:25.257793+0000 (oldest deadline 2024-09-21T00:28:50.557604+0000) 2024-09-21T00:28:57.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:57 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7[88260]: 2024-09-21T00:28:57.177+0000 7ff218e0a640 -1 osd.7 167 heartbeat_check: no reply from 172.21.15.31:6804 osd.4 since back 2024-09-21T00:28:33.859021+0000 front 2024-09-21T00:28:33.859068+0000 (oldest deadline 2024-09-21T00:28:56.659421+0000) 2024-09-21T00:28:58.547 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:58 smithi031 podman[97785]: 2024-09-21 00:28:58.282539101 +0000 UTC m=+5.557086173 container died 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-9e2ac70, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T00:28:58.812 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:58 smithi031 podman[97785]: 2024-09-21 00:28:58.627190747 +0000 UTC m=+5.901737809 container cleanup 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T00:28:58.812 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:58 smithi031 bash[97785]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7 2024-09-21T00:28:59.230 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:58 smithi031 podman[97801]: 2024-09-21 00:28:58.811813498 +0000 UTC m=+0.522355654 container remove 3a69d17a3c5896d894bfe5e1cc8ddb9e113cec57d2b73ac9b8ca7f48747b1331 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T00:28:59.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.229267265 +0000 UTC m=+0.111348761 container create 90557ff07d2c55ad632a41b92927c04c69a658141108752051caea22b0882310 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-9e2ac70, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T00:28:59.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.13495448 +0000 UTC m=+0.017035986 image pull cc9251b744a3faafca7c3095f79bfff81873afe823f31a03a11e015a34abcdbd quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736 2024-09-21T00:28:59.501 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.450656175 +0000 UTC m=+0.332737681 container init 90557ff07d2c55ad632a41b92927c04c69a658141108752051caea22b0882310 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-9e2ac70, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T00:28:59.502 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.454206764 +0000 UTC m=+0.336288261 container start 90557ff07d2c55ad632a41b92927c04c69a658141108752051caea22b0882310 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-9e2ac70) 2024-09-21T00:28:59.946 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.533670603 +0000 UTC m=+0.415752100 container attach 90557ff07d2c55ad632a41b92927c04c69a658141108752051caea22b0882310 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70) 2024-09-21T00:28:59.946 INFO:journalctl@ceph.osd.7.smithi031.stdout:Sep 21 00:28:59 smithi031 podman[97890]: 2024-09-21 00:28:59.646967957 +0000 UTC m=+0.529049453 container died 90557ff07d2c55ad632a41b92927c04c69a658141108752051caea22b0882310 (image=quay.ceph.io/ceph-ci/ceph:9e2ac70bb7aea67d34f0d50002344abbe7803736, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-9e2ac70, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T00:29:00.271 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@osd.7.service' 2024-09-21T00:29:00.311 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:29:00.311 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T00:29:00.311 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T00:29:00.311 DEBUG:teuthology.orchestra.run.smithi031:> sudo systemctl stop ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a 2024-09-21T00:29:00.632 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 systemd[1]: Stopping Ceph prometheus.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:29:00.899 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.631Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:29:00.899 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T00:29:00.899 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T00:29:00.899 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.632Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.633Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.633Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a[59538]: ts=2024-09-21T00:29:00.633Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T00:29:00.900 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 podman[97994]: 2024-09-21 00:29:00.642234462 +0000 UTC m=+0.105571605 container died 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:01.251 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 podman[97994]: 2024-09-21 00:29:00.997984196 +0000 UTC m=+0.461321345 container cleanup 93675685393e9219dd8c8e2a0d10d81366420ee5273751c70c7ab5ae73941a88 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:01.251 INFO:journalctl@ceph.prometheus.a.smithi031.stdout:Sep 21 00:29:00 smithi031 bash[97994]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-prometheus-a 2024-09-21T00:29:01.433 DEBUG:teuthology.orchestra.run.smithi031:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@prometheus.a.service' 2024-09-21T00:29:01.468 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T00:29:01.468 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T00:29:01.468 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-21T00:29:03.373 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:03 smithi018 systemd[1]: Stopping Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:29:03.637 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:03 smithi018 podman[123315]: 2024-09-21 00:29:03.372316016 +0000 UTC m=+0.104618336 container died b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:03.925 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:03 smithi018 podman[123315]: 2024-09-21 00:29:03.736537127 +0000 UTC m=+0.468839435 container cleanup b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:03.925 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:03 smithi018 bash[123315]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a 2024-09-21T00:29:04.347 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:29:04.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:29:04.347 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:29:04.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi018.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.18:8443: connect: connection refused" 2024-09-21T00:29:04.347 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:29:04.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi031.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.31:8443: connect: connection refused" 2024-09-21T00:29:04.348 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:04 smithi018 podman[123327]: 2024-09-21 00:29:04.029363259 +0000 UTC m=+0.656071465 container remove b74742d0347a7529a89053eb42f43a09f369c43c4515af77c91632533afae893 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:04.348 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:04 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:29:04.348 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:04 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T00:29:04.348 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:04 smithi018 systemd[1]: Stopped Ceph node-exporter.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:29:04.348 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 21 00:29:04 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.a.service: Consumed 4.654s CPU time. 2024-09-21T00:29:04.605 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 systemd[1]: Stopping Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:29:04.911 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a[58141]: ts=2024-09-21T00:29:04.599Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T00:29:04.911 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 podman[123434]: 2024-09-21 00:29:04.640445546 +0000 UTC m=+0.148501959 container died 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:05.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 podman[123434]: 2024-09-21 00:29:04.999357298 +0000 UTC m=+0.507413703 container cleanup 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:05.175 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:04 smithi018 bash[123434]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a 2024-09-21T00:29:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:05 smithi018 podman[123445]: 2024-09-21 00:29:05.275625794 +0000 UTC m=+0.628549059 container remove 503cf3565fc2b9c7e45cac5afbe9d2ffda774d9022e743506f5143a8488a3c94 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T00:29:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:05 smithi018 podman[123445]: 2024-09-21 00:29:05.351062712 +0000 UTC m=+0.703985934 volume remove 32167432dfd5ec252296abcbb8161d1a64627eb3f1445f1566a6738b49db906d 2024-09-21T00:29:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:05 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T00:29:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:05 smithi018 systemd[1]: Stopped Ceph alertmanager.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:29:05.675 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 21 00:29:05 smithi018 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@alertmanager.a.service: Consumed 2.245s CPU time. 2024-09-21T00:29:28.515 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-21T00:29:30.501 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:30 smithi031 systemd[1]: Stopping Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:29:30.784 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:30 smithi031 podman[98403]: 2024-09-21 00:29:30.526970871 +0000 UTC m=+0.242313759 container died 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:29:31.142 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:30 smithi031 podman[98403]: 2024-09-21 00:29:30.957780723 +0000 UTC m=+0.673123636 container cleanup 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:29:31.142 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:30 smithi031 bash[98403]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b 2024-09-21T00:29:31.410 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:31 smithi031 podman[98416]: 2024-09-21 00:29:31.142198589 +0000 UTC m=+0.618661760 container remove 2c775e5bd33dbd06ef8d855723ffa8040974b4cb007b3fd8165c0b3567945229 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T00:29:31.410 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:31 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T00:29:31.410 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:31 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T00:29:31.410 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:31 smithi031 systemd[1]: Stopped Ceph node-exporter.b for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:29:31.410 INFO:journalctl@ceph.node-exporter.b.smithi031.stdout:Sep 21 00:29:31 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@node-exporter.b.service: Consumed 4.495s CPU time. 2024-09-21T00:29:31.751 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:31 smithi031 systemd[1]: Stopping Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490... 2024-09-21T00:29:32.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=server t=2024-09-21T00:29:31.845586472Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-21T00:29:32.112 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=tracing t=2024-09-21T00:29:31.845985333Z level=info msg="Closing tracing" 2024-09-21T00:29:32.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:31 smithi031 ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a[61738]: logger=ticker t=2024-09-21T00:29:31.846012976Z level=info msg=stopped last_tick=2024-09-21T00:29:30Z 2024-09-21T00:29:32.113 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:31 smithi031 podman[98524]: 2024-09-21 00:29:31.85599335 +0000 UTC m=+0.132097051 container died 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:29:32.371 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 podman[98524]: 2024-09-21 00:29:32.195468117 +0000 UTC m=+0.471571808 container cleanup 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:29:32.371 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 bash[98524]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a 2024-09-21T00:29:32.624 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 podman[98535]: 2024-09-21 00:29:32.371234727 +0000 UTC m=+0.516535795 container remove 29b0278313a5e03f757395db32f2fa8ea604e91ce0108ada0fe96385dd48f5ba (image=quay.io/ceph/grafana:9.4.12, name=ceph-c1f9e122-77aa-11ef-baf5-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T00:29:33.001 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T00:29:33.001 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 systemd[1]: Stopped Ceph grafana.a for c1f9e122-77aa-11ef-baf5-efdc52797490. 2024-09-21T00:29:33.001 INFO:journalctl@ceph.grafana.a.smithi031.stdout:Sep 21 00:29:32 smithi031 systemd[1]: ceph-c1f9e122-77aa-11ef-baf5-efdc52797490@grafana.a.service: Consumed 11.106s CPU time. 2024-09-21T00:29:44.305 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T00:29:44.339 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T00:29:44.367 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T00:29:44.369 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi018/crash 2024-09-21T00:29:44.370 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash -- . 2024-09-21T00:29:44.431 INFO:teuthology.orchestra.run.smithi018.stderr:tar: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T00:29:44.432 INFO:teuthology.orchestra.run.smithi018.stderr:tar: Error is not recoverable: exiting now 2024-09-21T00:29:44.434 DEBUG:teuthology.misc:Transferring archived files from smithi031:/var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi031/crash 2024-09-21T00:29:44.435 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash -- . 2024-09-21T00:29:44.468 INFO:teuthology.orchestra.run.smithi031.stderr:tar: /var/lib/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T00:29:44.468 INFO:teuthology.orchestra.run.smithi031.stderr:tar: Error is not recoverable: exiting now 2024-09-21T00:29:44.469 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T00:29:44.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-09-21T00:29:44.503 INFO:tasks.cephadm:Compressing logs... 2024-09-21T00:29:44.503 DEBUG:teuthology.orchestra.run.smithi018:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T00:29:44.546 DEBUG:teuthology.orchestra.run.smithi031:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T00:29:44.572 INFO:teuthology.orchestra.run.smithi031.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.7.log 2024-09-21T00:29:44.572 INFO:teuthology.orchestra.run.smithi031.stderr:: No such file or directory 2024-09-21T00:29:44.572 INFO:teuthology.orchestra.run.smithi031.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-21T00:29:44.572 INFO:teuthology.orchestra.run.smithi031.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.4.log 2024-09-21T00:29:44.573 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.7.log: /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-21T00:29:44.573 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-21T00:29:44.573 INFO:teuthology.orchestra.run.smithi031.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.6.log 2024-09-21T00:29:44.573 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log: /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.5.log 2024-09-21T00:29:44.576 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log 2024-09-21T00:29:44.576 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.x.log 2024-09-21T00:29:44.576 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log: 89.6% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.0.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:: No such file or directory 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/tcmu-runner.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.1.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr: 86.8% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/tcmu-runner.log.gz 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.c.log 2024-09-21T00:29:44.578 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-21T00:29:44.579 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.a.log 2024-09-21T00:29:44.579 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.2.log 2024-09-21T00:29:44.579 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.y.log 2024-09-21T00:29:44.579 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.2.log: 93.9% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-21T00:29:44.581 INFO:teuthology.orchestra.run.smithi031.stderr: 89.1% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-21T00:29:44.581 INFO:teuthology.orchestra.run.smithi031.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.b.log 2024-09-21T00:29:44.582 INFO:teuthology.orchestra.run.smithi031.stderr: 91.1% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-21T00:29:44.582 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi031.jzvksl.log 2024-09-21T00:29:44.582 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T00:29:44.582 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi031.jzvksl.log: 76.2% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi031.jzvksl.log.gz 2024-09-21T00:29:44.583 INFO:teuthology.orchestra.run.smithi018.stderr: 94.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-21T00:29:44.586 INFO:teuthology.orchestra.run.smithi031.stderr:/var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-21T00:29:44.587 INFO:teuthology.orchestra.run.smithi018.stderr: 90.8% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-21T00:29:44.588 INFO:teuthology.orchestra.run.smithi018.stderr:gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi018.kionxd.log 2024-09-21T00:29:44.589 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log 2024-09-21T00:29:44.590 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi018.kionxd.log: gzip -5 --verbose -- /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.3.log 2024-09-21T00:29:44.590 INFO:teuthology.orchestra.run.smithi018.stderr: 76.3% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-client.rgw.foo.smithi018.kionxd.log.gz 2024-09-21T00:29:44.597 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log: 94.0% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-21T00:29:44.597 INFO:teuthology.orchestra.run.smithi018.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T00:29:44.605 INFO:teuthology.orchestra.run.smithi031.stderr: 90.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T00:29:44.615 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T00:29:44.699 INFO:teuthology.orchestra.run.smithi031.stderr: 90.1% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T00:29:44.930 INFO:teuthology.orchestra.run.smithi018.stderr: 89.5% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T00:29:45.225 INFO:teuthology.orchestra.run.smithi031.stderr: 92.5% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.b.log.gz 2024-09-21T00:29:45.386 INFO:teuthology.orchestra.run.smithi018.stderr: 92.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.c.log.gz 2024-09-21T00:29:46.044 INFO:teuthology.orchestra.run.smithi018.stderr: 93.3% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.2.log.gz 2024-09-21T00:29:46.122 INFO:teuthology.orchestra.run.smithi018.stderr: 91.0% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-mon.a.log.gz 2024-09-21T00:29:46.137 INFO:teuthology.orchestra.run.smithi031.stderr: 93.1% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.6.log.gz 2024-09-21T00:29:46.612 INFO:teuthology.orchestra.run.smithi018.stderr: 93.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.1.log.gz 2024-09-21T00:29:46.656 INFO:teuthology.orchestra.run.smithi031.stderr: 93.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.5.log.gz 2024-09-21T00:29:46.681 INFO:teuthology.orchestra.run.smithi031.stderr: 93.5% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.4.log.gz 2024-09-21T00:29:46.753 INFO:teuthology.orchestra.run.smithi031.stderr: 93.6% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.7.log.gz 2024-09-21T00:29:46.755 INFO:teuthology.orchestra.run.smithi031.stderr: 2024-09-21T00:29:46.755 INFO:teuthology.orchestra.run.smithi031.stderr:real 0m2.194s 2024-09-21T00:29:46.755 INFO:teuthology.orchestra.run.smithi031.stderr:user 0m8.461s 2024-09-21T00:29:46.755 INFO:teuthology.orchestra.run.smithi031.stderr:sys 0m0.307s 2024-09-21T00:29:46.842 INFO:teuthology.orchestra.run.smithi018.stderr: 93.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.0.log.gz 2024-09-21T00:29:47.295 INFO:teuthology.orchestra.run.smithi018.stderr: 93.4% -- replaced with /var/log/ceph/c1f9e122-77aa-11ef-baf5-efdc52797490/ceph-osd.3.log.gz 2024-09-21T00:29:47.297 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T00:29:47.297 INFO:teuthology.orchestra.run.smithi018.stderr:real 0m2.737s 2024-09-21T00:29:47.297 INFO:teuthology.orchestra.run.smithi018.stderr:user 0m10.853s 2024-09-21T00:29:47.298 INFO:teuthology.orchestra.run.smithi018.stderr:sys 0m0.383s 2024-09-21T00:29:47.298 INFO:tasks.cephadm:Archiving logs... 2024-09-21T00:29:47.298 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi018/log 2024-09-21T00:29:47.299 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T00:29:48.686 DEBUG:teuthology.misc:Transferring archived files from smithi031:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi031/log 2024-09-21T00:29:48.687 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T00:29:49.844 INFO:tasks.cephadm:Removing cluster... 2024-09-21T00:29:49.844 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 --force 2024-09-21T00:29:50.144 DEBUG:teuthology.orchestra.run.smithi031:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c1f9e122-77aa-11ef-baf5-efdc52797490 --force 2024-09-21T00:29:50.442 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T00:29:50.442 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T00:29:50.460 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T00:29:50.476 INFO:tasks.cephadm:Teardown complete 2024-09-21T00:29:50.476 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T00:29:50.486 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T00:29:50.486 DEBUG:teuthology.orchestra.run.smithi018:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T00:29:50.502 DEBUG:teuthology.orchestra.run.smithi031:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T00:29:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:bash: line 1: ntpq: command not found 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:^* hv01.front.sepia.ceph.com 3 6 377 53 +287us[ +297us] +/- 34ms 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 -1387us[-1387us] +/- 55ms 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +830us[ +830us] +/- 71ms 2024-09-21T00:29:50.520 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T00:29:50.530 INFO:teuthology.orchestra.run.smithi031.stderr:bash: line 1: ntpq: command not found 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:=============================================================================== 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:^* hv01.front.sepia.ceph.com 3 6 377 50 +310us[ +274us] +/- 34ms 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 -1362us[-1398us] +/- 55ms 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 53 +894us[ +859us] +/- 71ms 2024-09-21T00:29:50.534 INFO:teuthology.orchestra.run.smithi031.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T00:29:50.535 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T00:29:50.544 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T00:29:50.545 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T00:29:50.553 DEBUG:teuthology.orchestra.run.smithi018:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T00:29:50.654 DEBUG:teuthology.orchestra.run.smithi031:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T00:29:50.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-21T00:29:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T00:29:50.765 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-21T00:29:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T00:29:50.795 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T00:29:50.804 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T00:29:50.849 INFO:teuthology.task.internal:Duration was 2750.791985 seconds 2024-09-21T00:29:50.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T00:29:50.858 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T00:29:50.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T00:29:50.860 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T00:29:50.899 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T00:29:50.901 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T00:29:51.220 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T00:29:51.221 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T00:29:51.221 DEBUG:teuthology.orchestra.run.smithi018:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T00:29:51.282 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi031.front.sepia.ceph.com 2024-09-21T00:29:51.283 DEBUG:teuthology.orchestra.run.smithi031:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T00:29:51.307 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T00:29:51.307 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T00:29:51.326 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T00:29:51.385 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T00:29:51.386 DEBUG:teuthology.orchestra.run.smithi018:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T00:29:51.395 DEBUG:teuthology.orchestra.run.smithi031:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T00:29:51.976 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T00:29:51.992 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T00:29:51.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T00:29:52.018 DEBUG:teuthology.orchestra.run.smithi031:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T00:29:52.044 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T00:29:52.053 DEBUG:teuthology.orchestra.run.smithi018:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T00:29:52.060 DEBUG:teuthology.orchestra.run.smithi031:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T00:29:52.082 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = core 2024-09-21T00:29:52.110 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = core 2024-09-21T00:29:52.132 DEBUG:teuthology.orchestra.run.smithi018:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T00:29:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T00:29:52.165 DEBUG:teuthology.orchestra.run.smithi031:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T00:29:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T00:29:52.187 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T00:29:52.199 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T00:29:52.200 DEBUG:teuthology.misc:Transferring archived files from smithi018:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi018 2024-09-21T00:29:52.200 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T00:29:52.260 DEBUG:teuthology.misc:Transferring archived files from smithi031:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-20_21:08:02-orch-squid-distro-default-smithi/7913105/remote/smithi031 2024-09-21T00:29:52.260 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T00:29:52.302 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T00:29:52.302 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T00:29:52.304 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T00:29:52.352 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T00:29:52.363 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T00:29:52.363 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T00:29:52.373 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T00:29:52.373 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T00:29:52.376 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T00:29:52.389 INFO:teuthology.orchestra.run.smithi018.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 00:29 /home/ubuntu/cephtest 2024-09-21T00:29:52.407 INFO:teuthology.orchestra.run.smithi031.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 00:29 /home/ubuntu/cephtest 2024-09-21T00:29:52.408 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T00:29:52.418 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T00:29:52.438 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2750.791985273361 owner: scheduled_teuthology@teuthology success: true 2024-09-21T00:29:52.438 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T00:29:52.520 INFO:teuthology.run:pass